slicetest 0.7.0 → 0.8.0

package/dist/yaml.js

@@ -1,15 +1,18 @@
 import { isMap, isSeq, LineCounter, parseDocument } from "yaml";
+import { WEBHOOK_PROVIDERS as PROVIDERS } from "./webhook.js";
 const KINDS = {
     stub: ["on", "graphql", "when", "reply", "sequence", "networkError", "times", "delay", "optional"],
-    request: ["headers", "query", "json", "form", "body", "graphql", "follow", "auth", "webhook", "concurrency", "expect", "capture"],
+    request: ["headers", "query", "json", "form", "multipart", "body", "graphql", "follow", "auth", "webhook", "concurrency", "expect", "capture", "within", "every", "timeout"],
     submit: ["form", "fields", "headers", "follow", "expect", "capture"],
     insert: ["rows", "capture"],
     sql: ["params", "expect", "capture", "within"],
     db: ["where", "orderBy", "expect", "capture", "within"],
     received: ["call", "graphql", "when", "times", "within"],
-    changes: ["within"],
+    changes: ["within", "ignore"],
     log: ["from", "within"],
     checkpoint: [],
+    set: [],
+    order: ["within"],
     snapshot: ["mask"],
     mail: ["times", "within", "capture"],
     make: ["rows", "count", "capture"],
@@ -22,13 +25,13 @@ const EXPECT_KEYS = {
     sql: ["rows", "count"],
     db: ["rows", "count"],
 };
-const CONDITION_KEYS = ["query", "headers", "json", "body", "variables"];
-const WEBHOOK_KEYS = ["provider", "secret", "event", "stale", "invalidSignature"];
-const WEBHOOK_PROVIDERS = ["stripe", "github", "slack", "shopify", "standard"];
+const CONDITION_KEYS = ["query", "headers", "json", "form", "body", "variables"];
+const WEBHOOK_KEYS = ["provider", "secret", "event", "stale", "invalidSignature", "url"];
+const WEBHOOK_PROVIDERS = PROVIDERS;
 const MAIL_KEYS = ["to", "from", "subject", "text", "html"];
 const CHANGE_KEYS = ["inserted", "updated", "deleted"];
-const RESPONSE_KEYS = ["status", "headers", "body", "sse", "data", "errors"];
-const SCENARIO_KEYS = ["name", "steps", "each", "skip", "only", "timeout"];
+const RESPONSE_KEYS = ["status", "headers", "body", "file", "sse", "data", "errors"];
+const SCENARIO_KEYS = ["name", "steps", "each", "skip", "only", "timeout", "tags"];
 const CALL = /^([A-Za-z]+|\*)\s+(\/\S*)$/;
 /** A request may also go to a captured URL of the app, e.g. a link from a mail: `GET {{link}}`. */
 const REQUEST = /^[A-Za-z]+\s+(\/\S*|\{\{[^}]+\}\}\S*)$/;
@@ -124,6 +127,9 @@ export function parseScenarioFile(text, file) {
             }
             if (raw.timeout !== undefined && typeof raw.timeout !== "number")
                 fail(node, "`timeout` must be a number of milliseconds");
+            if (raw.tags !== undefined && !(Array.isArray(raw.tags) && raw.tags.every((t) => typeof t === "string" && /^[^\s,!]+$/.test(t)))) {
+                fail(node.get("tags", true), "`tags` must be a list of words without spaces, commas or `!`, e.g. [smoke, payments]");
+            }
             return {
                 name: raw.name,
                 line: lineOf(node),
@@ -131,6 +137,7 @@ export function parseScenarioFile(text, file) {
                 skip: raw.skip === true,
                 only: raw.only === true,
                 timeout: raw.timeout,
+                tags: raw.tags,
                 steps: stepsOf(node.get("steps", true), `scenario "${raw.name}": steps`),
             };
         }),
@@ -156,6 +163,8 @@ function parseStep(node, fail, lineOf) {
             fail(at(key), `unknown key "${key}" in a ${kind} step (allowed: ${[...allowed].join(", ")})`);
     }
     if (kind === "changes") {
+        if (raw.ignore !== undefined && !(Array.isArray(raw.ignore) && raw.ignore.every((c) => typeof c === "string" && c)))
+            fail(at("ignore"), "`ignore` must be a list of columns or tables, e.g. [updated_at, sessions.*]");
         const tables = raw.changes;
         if (!tables || typeof tables !== "object" || Array.isArray(tables))
             fail(at(kind), "`changes:` must map table names to { inserted, updated, deleted }");
@@ -189,8 +198,9 @@ function parseStep(node, fail, lineOf) {
                 fail(at("fields"), "`fields` must map field names to values, e.g. { email: a@b.test }");
             for (const [k, v] of Object.entries(fields)) {
                 const scalar = (x) => typeof x === "string" || typeof x === "number";
-                if (!(scalar(v) || typeof v === "boolean" || (Array.isArray(v) && v.every(scalar))))
-                    fail(at("fields"), `field "${k}" must be a string, number, true / false or a list`);
+                const isFile = !!v && typeof v === "object" && !Array.isArray(v) && Object.keys(v).every((x) => ["file", "content", "filename", "type"].includes(x)) && ("file" in v) !== ("content" in v);
+                if (!(scalar(v) || typeof v === "boolean" || (Array.isArray(v) && v.every(scalar)) || isFile))
+                    fail(at("fields"), `field "${k}" must be a string, number, true / false, a list, or a file ({ file: path } or { content, filename })`);
             }
         }
     }
@@ -200,6 +210,42 @@ function parseStep(node, fail, lineOf) {
         if (raw.with !== undefined && (!raw.with || typeof raw.with !== "object" || Array.isArray(raw.with)))
             fail(at("with"), "`with` maps params to values, e.g. { email: a@b.test }");
     }
+    else if (kind === "order") {
+        const list = raw.order;
+        if (!Array.isArray(list) || list.length < 2)
+            fail(at(kind), "`order:` lists at least two calls, e.g. [\"stripe POST /v1/charges\", \"mail POST /send\"]");
+        for (const item of list) {
+            if (typeof item === "string") {
+                if (!/^\S+\s+([A-Za-z]+|\*)\s+\/\S*$/.test(item))
+                    fail(at(kind), `"${item}" should be "<stub> METHOD /path", e.g. "mail POST /send"`);
+                continue;
+            }
+            if (!item || typeof item !== "object" || Array.isArray(item))
+                fail(at(kind), "each `order` entry is \"<stub> METHOD /path\" or { stub, call, when }");
+            const o = item;
+            for (const k of Object.keys(o))
+                if (!["stub", "call", "when"].includes(k))
+                    fail(at(kind), `unknown key "${k}" in an order entry (allowed: stub, call, when)`);
+            if (typeof o.stub !== "string" || !o.stub)
+                fail(at(kind), "an order entry needs `stub`");
+            if (typeof o.call !== "string" || !CALL.test(o.call))
+                fail(at(kind), `an order entry's \`call\` must look like "POST /path", got ${JSON.stringify(o.call)}`);
+            if (o.when !== undefined) {
+                if (!o.when || typeof o.when !== "object" || Array.isArray(o.when))
+                    fail(at(kind), "`when` in an order entry must be a mapping");
+                for (const k of Object.keys(o.when))
+                    if (!CONDITION_KEYS.includes(k))
+                        fail(at(kind), `unknown key "${k}" in when (allowed: ${CONDITION_KEYS.join(", ")})`);
+            }
+        }
+    }
+    else if (kind === "set") {
+        if (!raw.set || typeof raw.set !== "object" || Array.isArray(raw.set) || Object.keys(raw.set).length === 0)
+            fail(at(kind), "`set:` maps variable names to values, e.g. { orderId: \"{{$uuid}}\" }");
+        for (const k of Object.keys(raw.set))
+            if (!/^[A-Za-z_][\w]*$/.test(k))
+                fail(at(kind), `"${k}" isn't a variable name (letters, digits and _)`);
+    }
     else if (kind === "checkpoint") {
         if (raw.checkpoint !== true)
             fail(at(kind), "use `checkpoint: true`");
@@ -264,7 +310,13 @@ function parseStep(node, fail, lineOf) {
             for (const [where, r] of [["reply", raw.reply], ...(Array.isArray(raw.sequence) ? raw.sequence : []).map((r) => ["sequence", r])]) {
                 if (!r || typeof r !== "object")
                     continue;
-                const { body, sse, data, errors } = r;
+                const { body, sse, data, errors, file } = r;
+                if (file !== undefined) {
+                    if (typeof file !== "string" || !file)
+                        fail(at(where), "`file` is a path relative to the scenario file, e.g. fixtures/charge.json");
+                    if (body !== undefined || sse !== undefined || data !== undefined || errors !== undefined)
+                        fail(at(where), "a reply has `file` instead of `body`, `sse`, `data` or `errors`");
+                }
                 if (data !== undefined || errors !== undefined) {
                     if (raw.graphql === undefined)
                         fail(at(where), "`data` and `errors` answer a GraphQL operation: use them with `graphql: <operation>`, or send `body`");
@@ -292,8 +344,27 @@ function parseStep(node, fail, lineOf) {
         }
         case "request":
             call("request", REQUEST);
-            if (["json", "form", "body", "graphql"].filter((k) => raw[k] !== undefined).length > 1)
-                fail(node, "use only one of json / form / body / graphql");
+            if (["json", "form", "multipart", "body", "graphql"].filter((k) => raw[k] !== undefined).length > 1)
+                fail(node, "use only one of json / form / multipart / body / graphql");
+            if (raw.multipart !== undefined) {
+                const m = raw.multipart;
+                if (!m || typeof m !== "object" || Array.isArray(m))
+                    fail(at("multipart"), "`multipart` maps field names to values, e.g. { name: ada, avatar: { file: ada.png } }");
+                for (const [k, v] of Object.entries(m)) {
+                    for (const part of Array.isArray(v) ? v : [v]) {
+                        if (part && typeof part === "object") {
+                            const keys = Object.keys(part);
+                            const bad = keys.find((x) => !["file", "content", "filename", "type"].includes(x));
+                            if (bad)
+                                fail(at("multipart"), `unknown key "${bad}" in multipart field "${k}" (allowed: file, content, filename, type)`);
+                            if (("file" in part) === ("content" in part))
+                                fail(at("multipart"), `multipart field "${k}" needs one of \`file\` (a path relative to the scenario file) or \`content\``);
+                        }
+                        else if (part === null)
+                            fail(at("multipart"), `multipart field "${k}" has no value`);
+                    }
+                }
+            }
             if (raw.graphql !== undefined) {
                 const g = raw.graphql;
                 const ok = (typeof g === "string" && g.trim()) || (g && typeof g === "object" && !Array.isArray(g) && typeof g.query === "string" && Object.keys(g).every((k) => ["query", "variables", "operationName"].includes(k)));
@@ -318,6 +389,14 @@ function parseStep(node, fail, lineOf) {
                 fail(at("auth"), "`auth` must be true or the token's claims, e.g. { sub: u1, roles: [admin] }");
             if (raw.concurrency !== undefined && !(Number.isInteger(raw.concurrency) && raw.concurrency >= 1))
                 fail(at("concurrency"), "`concurrency` must be a positive whole number");
+            if (raw.timeout !== undefined && !(typeof raw.timeout === "number" && raw.timeout > 0))
+                fail(at("timeout"), "`timeout` must be a positive number of milliseconds");
+            if (raw.every !== undefined && !(typeof raw.every === "number" && raw.every > 0))
+                fail(at("every"), "`every` must be a positive number of milliseconds");
+            if (raw.every !== undefined && raw.within === undefined)
+                fail(at("every"), "`every` sets the pause between attempts of a `within` step; add `within: <ms>`");
+            if (raw.within !== undefined && raw.concurrency !== undefined)
+                fail(at("within"), "`within` repeats one request until it passes; it can't be combined with `concurrency`");
             {
                 const q = raw.expect?.queries;
                 if (q !== undefined && !(Number.isInteger(q) && q >= 0))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "slicetest",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Framework-agnostic slice tests: drive a real app process over HTTP with a real Postgres, MySQL or SQLite and stubbed outbound APIs, checked against OpenAPI contracts.",
   "keywords": [
     "testing",

package/schema/scenario.schema.json

@@ -86,6 +86,14 @@
             "items": {
               "$ref": "#/$defs/step"
             }
+          },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "pattern": "^[^\\s,!]+$"
+            },
+            "description": "Labels to select scenarios by: npx slicetest --tag smoke (or !slow to leave some out)"
           }
         }
       }
@@ -127,6 +135,10 @@
                 "json": {
                   "description": "Subset of the request JSON"
                 },
+                "form": {
+                  "type": "object",
+                  "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                },
                 "body": {},
                 "variables": {
                   "description": "Subset of the GraphQL variables (with `graphql`)"
@@ -149,6 +161,10 @@
                 "body": {
                   "description": "String, or anything else to send as JSON. May use {{call.params.x}}, {{call.json.x}}, {{call.query.x}}"
                 },
+                "file": {
+                  "type": "string",
+                  "description": "Answer with this file's content, relative to the scenario file: .json and .yaml are sent as JSON (and may use {{call.*}}), other files as they are, with a content type from the extension"
+                },
                 "sse": {
                   "type": "array",
                   "description": "A streamed reply (Server-Sent Events), instead of body: a list of { event, data, id }. Data that isn't a string is sent as JSON",
@@ -221,6 +237,30 @@
                       "sse",
                       "errors"
                     ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "body"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "sse"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "data"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "errors"
+                    ]
                   }
                 ]
               }
@@ -243,6 +283,10 @@
                   "body": {
                     "description": "String, or anything else to send as JSON. May use {{call.params.x}}, {{call.json.x}}, {{call.query.x}}"
                   },
+                  "file": {
+                    "type": "string",
+                    "description": "Answer with this file's content, relative to the scenario file: .json and .yaml are sent as JSON (and may use {{call.*}}), other files as they are, with a content type from the extension"
+                  },
                   "sse": {
                     "type": "array",
                     "description": "A streamed reply (Server-Sent Events), instead of body: a list of { event, data, id }. Data that isn't a string is sent as JSON",
@@ -315,6 +359,30 @@
                         "sse",
                         "errors"
                       ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "body"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "sse"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "data"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "errors"
+                      ]
                     }
                   ]
                 }
@@ -389,6 +457,103 @@
             "form": {
               "type": "object"
             },
+            "multipart": {
+              "type": "object",
+              "description": "A multipart/form-data body: fields, and files as { file: avatar.png } (relative to the scenario file) or { content, filename, type }",
+              "additionalProperties": {
+                "oneOf": [
+                  {
+                    "oneOf": [
+                      {
+                        "type": [
+                          "string",
+                          "number",
+                          "boolean"
+                        ]
+                      },
+                      {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                          "file": {
+                            "type": "string",
+                            "description": "Path relative to the scenario file"
+                          },
+                          "content": {
+                            "description": "Inline file content; objects are sent as JSON"
+                          },
+                          "filename": {
+                            "type": "string"
+                          },
+                          "type": {
+                            "type": "string",
+                            "description": "Content type; from the file extension by default"
+                          }
+                        },
+                        "oneOf": [
+                          {
+                            "required": [
+                              "file"
+                            ]
+                          },
+                          {
+                            "required": [
+                              "content"
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "oneOf": [
+                        {
+                          "type": [
+                            "string",
+                            "number",
+                            "boolean"
+                          ]
+                        },
+                        {
+                          "type": "object",
+                          "additionalProperties": false,
+                          "properties": {
+                            "file": {
+                              "type": "string",
+                              "description": "Path relative to the scenario file"
+                            },
+                            "content": {
+                              "description": "Inline file content; objects are sent as JSON"
+                            },
+                            "filename": {
+                              "type": "string"
+                            },
+                            "type": {
+                              "type": "string",
+                              "description": "Content type; from the file extension by default"
+                            }
+                          },
+                          "oneOf": [
+                            {
+                              "required": [
+                                "file"
+                              ]
+                            },
+                            {
+                              "required": [
+                                "content"
+                              ]
+                            }
+                          ]
+                        }
+                      ]
+                    }
+                  }
+                ]
+              }
+            },
             "body": {
               "type": "string"
             },
@@ -423,7 +588,14 @@
                         "github",
                         "slack",
                         "shopify",
-                        "standard"
+                        "standard",
+                        "line",
+                        "paddle",
+                        "linear",
+                        "gitlab",
+                        "zoom",
+                        "twitch",
+                        "twilio"
                       ]
                     },
                     {
@@ -467,6 +639,10 @@
                 "invalidSignature": {
                   "type": "boolean",
                   "description": "Signed with a different secret"
+                },
+                "url": {
+                  "type": "string",
+                  "description": "The URL the app validates against, for providers that sign it (twilio). Default: the app's address and the request path"
                 }
               }
             },
@@ -553,6 +729,21 @@
                   }
                 }
               ]
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Repeat the request until `expect` passes, for up to this many milliseconds (polling a job the app finishes later)"
+            },
+            "every": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "With `within`: milliseconds between attempts. Default 200"
+            },
+            "timeout": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Fail when the app hasn't answered within this many milliseconds (overrides http.timeout)"
             }
           },
           "not": {
@@ -574,8 +765,43 @@
                   "graphql",
                   "body"
                 ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "json"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "form"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "body"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "graphql"
+                ]
+              },
+              {
+                "required": [
+                  "within",
+                  "concurrency"
+                ]
               }
             ]
+          },
+          "dependentRequired": {
+            "every": [
+              "within"
+            ]
           }
         },
         {
@@ -610,7 +836,7 @@
             },
             "fields": {
               "type": "object",
-              "description": "Values typed into the form by field name; true / false / a list of values check checkboxes and radios",
+              "description": "Values typed into the form by field name; true / false / a list of values check checkboxes and radios; { file: path } uploads a file through a file input",
               "additionalProperties": {
                 "oneOf": [
                   {
@@ -628,6 +854,36 @@
                         "number"
                       ]
                     }
+                  },
+                  {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                      "file": {
+                        "type": "string",
+                        "description": "Path relative to the scenario file"
+                      },
+                      "content": {},
+                      "filename": {
+                        "type": "string"
+                      },
+                      "type": {
+                        "type": "string"
+                      }
+                    },
+                    "oneOf": [
+                      {
+                        "required": [
+                          "file"
+                        ]
+                      },
+                      {
+                        "required": [
+                          "content"
+                        ]
+                      }
+                    ],
+                    "description": "A file for a file input"
                   }
                 ]
               }
@@ -940,6 +1196,10 @@
                 "json": {
                   "description": "Subset of the request JSON"
                 },
+                "form": {
+                  "type": "object",
+                  "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                },
                 "body": {},
                 "variables": {
                   "description": "Subset of the GraphQL variables (with `graphql`)"
@@ -1038,6 +1298,105 @@
               "type": "integer",
               "minimum": 1,
               "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            },
+            "ignore": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Columns or tables to leave out: updated_at (in every table), orders.synced_at, sessions.*"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "set"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "set": {
+              "type": "object",
+              "minProperties": 1,
+              "propertyNames": {
+                "pattern": "^[A-Za-z_]\\w*$"
+              },
+              "description": "Variables for later steps, e.g. { orderId: \"{{$uuid}}\", expires: \"{{$now+1d}}\" }"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "order"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "order": {
+              "type": "array",
+              "minItems": 2,
+              "description": "Calls to stubs in the order they must have happened (others may come in between): \"<stub> METHOD /path\" or { stub, call, when }",
+              "items": {
+                "oneOf": [
+                  {
+                    "type": "string",
+                    "pattern": "^\\S+\\s+([A-Za-z]+|\\*)\\s+/\\S*$"
+                  },
+                  {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": [
+                      "stub",
+                      "call"
+                    ],
+                    "properties": {
+                      "stub": {
+                        "type": "string"
+                      },
+                      "call": {
+                        "type": "string",
+                        "pattern": "^([A-Za-z]+|\\*)\\s+/\\S*$"
+                      },
+                      "when": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                          "query": {
+                            "type": "object"
+                          },
+                          "headers": {
+                            "type": "object"
+                          },
+                          "json": {
+                            "description": "Subset of the request JSON"
+                          },
+                          "form": {
+                            "type": "object",
+                            "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                          },
+                          "body": {},
+                          "variables": {
+                            "description": "Subset of the GraphQL variables (with `graphql`)"
+                          }
+                        }
+                      }
+                    }
+                  }
+                ]
+              }
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
             }
           }
         },