slicetest 0.6.1 → 0.8.0

package/schema/scenario.schema.json

@@ -15,6 +15,41 @@
         "$ref": "#/$defs/step"
       }
     },
+    "define": {
+      "type": "object",
+      "description": "Named step lists that `use:` steps run. A list of steps, or { params, steps } for values given with `with:`",
+      "additionalProperties": {
+        "oneOf": [
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/$defs/step"
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "steps"
+            ],
+            "properties": {
+              "params": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "steps": {
+                "type": "array",
+                "items": {
+                  "$ref": "#/$defs/step"
+                }
+              }
+            }
+          }
+        ]
+      }
+    },
     "scenarios": {
       "type": "array",
       "minItems": 1,
@@ -51,6 +86,14 @@
             "items": {
               "$ref": "#/$defs/step"
             }
+          },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "pattern": "^[^\\s,!]+$"
+            },
+            "description": "Labels to select scenarios by: npx slicetest --tag smoke (or !slow to leave some out)"
           }
         }
       }
@@ -62,8 +105,7 @@
         {
           "type": "object",
           "required": [
-            "stub",
-            "on"
+            "stub"
           ],
           "additionalProperties": false,
           "properties": {
@@ -93,7 +135,14 @@
                 "json": {
                   "description": "Subset of the request JSON"
                 },
-                "body": {}
+                "form": {
+                  "type": "object",
+                  "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                },
+                "body": {},
+                "variables": {
+                  "description": "Subset of the GraphQL variables (with `graphql`)"
+                }
               }
             },
             "reply": {
@@ -112,6 +161,10 @@
                 "body": {
                   "description": "String, or anything else to send as JSON. May use {{call.params.x}}, {{call.json.x}}, {{call.query.x}}"
                 },
+                "file": {
+                  "type": "string",
+                  "description": "Answer with this file's content, relative to the scenario file: .json and .yaml are sent as JSON (and may use {{call.*}}), other files as they are, with a content type from the extension"
+                },
                 "sse": {
                   "type": "array",
                   "description": "A streamed reply (Server-Sent Events), instead of body: a list of { event, data, id }. Data that isn't a string is sent as JSON",
@@ -131,12 +184,84 @@
                       }
                     }
                   }
+                },
+                "data": {
+                  "description": "GraphQL `data` (with `graphql`): the reply's body is { data, errors }"
+                },
+                "errors": {
+                  "type": "array",
+                  "description": "GraphQL errors (with `graphql`): messages or { message, path, extensions }",
+                  "items": {
+                    "oneOf": [
+                      {
+                        "type": "string"
+                      },
+                      {
+                        "type": "object",
+                        "required": [
+                          "message"
+                        ]
+                      }
+                    ]
+                  }
                 }
               },
               "not": {
-                "required": [
-                  "body",
-                  "sse"
+                "anyOf": [
+                  {
+                    "required": [
+                      "body",
+                      "sse"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "body",
+                      "data"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "body",
+                      "errors"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "sse",
+                      "data"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "sse",
+                      "errors"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "body"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "sse"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "data"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "file",
+                      "errors"
+                    ]
+                  }
                 ]
               }
             },
@@ -158,6 +283,10 @@
                   "body": {
                     "description": "String, or anything else to send as JSON. May use {{call.params.x}}, {{call.json.x}}, {{call.query.x}}"
                   },
+                  "file": {
+                    "type": "string",
+                    "description": "Answer with this file's content, relative to the scenario file: .json and .yaml are sent as JSON (and may use {{call.*}}), other files as they are, with a content type from the extension"
+                  },
                   "sse": {
                     "type": "array",
                     "description": "A streamed reply (Server-Sent Events), instead of body: a list of { event, data, id }. Data that isn't a string is sent as JSON",
@@ -177,12 +306,84 @@
                         }
                       }
                     }
+                  },
+                  "data": {
+                    "description": "GraphQL `data` (with `graphql`): the reply's body is { data, errors }"
+                  },
+                  "errors": {
+                    "type": "array",
+                    "description": "GraphQL errors (with `graphql`): messages or { message, path, extensions }",
+                    "items": {
+                      "oneOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "object",
+                          "required": [
+                            "message"
+                          ]
+                        }
+                      ]
+                    }
                   }
                 },
                 "not": {
-                  "required": [
-                    "body",
-                    "sse"
+                  "anyOf": [
+                    {
+                      "required": [
+                        "body",
+                        "sse"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "body",
+                        "data"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "body",
+                        "errors"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "sse",
+                        "data"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "sse",
+                        "errors"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "body"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "sse"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "data"
+                      ]
+                    },
+                    {
+                      "required": [
+                        "file",
+                        "errors"
+                      ]
+                    }
                   ]
                 }
               },
@@ -198,8 +399,38 @@
             "delay": {
               "type": "integer",
               "minimum": 0
+            },
+            "graphql": {
+              "type": "string",
+              "description": "A GraphQL operation (its operationName, or the name in the document), instead of METHOD /path. Matches POST and GET GraphQL requests to any path"
+            },
+            "optional": {
+              "type": "boolean",
+              "description": "The app may not call this route: strictStubs doesn't report it as unused"
             }
-          }
+          },
+          "oneOf": [
+            {
+              "required": [
+                "on"
+              ],
+              "not": {
+                "required": [
+                  "graphql"
+                ]
+              }
+            },
+            {
+              "required": [
+                "graphql"
+              ],
+              "not": {
+                "required": [
+                  "on"
+                ]
+              }
+            }
+          ]
         },
         {
           "type": "object",
@@ -226,6 +457,103 @@
             "form": {
               "type": "object"
             },
+            "multipart": {
+              "type": "object",
+              "description": "A multipart/form-data body: fields, and files as { file: avatar.png } (relative to the scenario file) or { content, filename, type }",
+              "additionalProperties": {
+                "oneOf": [
+                  {
+                    "oneOf": [
+                      {
+                        "type": [
+                          "string",
+                          "number",
+                          "boolean"
+                        ]
+                      },
+                      {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                          "file": {
+                            "type": "string",
+                            "description": "Path relative to the scenario file"
+                          },
+                          "content": {
+                            "description": "Inline file content; objects are sent as JSON"
+                          },
+                          "filename": {
+                            "type": "string"
+                          },
+                          "type": {
+                            "type": "string",
+                            "description": "Content type; from the file extension by default"
+                          }
+                        },
+                        "oneOf": [
+                          {
+                            "required": [
+                              "file"
+                            ]
+                          },
+                          {
+                            "required": [
+                              "content"
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "oneOf": [
+                        {
+                          "type": [
+                            "string",
+                            "number",
+                            "boolean"
+                          ]
+                        },
+                        {
+                          "type": "object",
+                          "additionalProperties": false,
+                          "properties": {
+                            "file": {
+                              "type": "string",
+                              "description": "Path relative to the scenario file"
+                            },
+                            "content": {
+                              "description": "Inline file content; objects are sent as JSON"
+                            },
+                            "filename": {
+                              "type": "string"
+                            },
+                            "type": {
+                              "type": "string",
+                              "description": "Content type; from the file extension by default"
+                            }
+                          },
+                          "oneOf": [
+                            {
+                              "required": [
+                                "file"
+                              ]
+                            },
+                            {
+                              "required": [
+                                "content"
+                              ]
+                            }
+                          ]
+                        }
+                      ]
+                    }
+                  }
+                ]
+              }
+            },
             "body": {
               "type": "string"
             },
@@ -260,7 +588,14 @@
                         "github",
                         "slack",
                         "shopify",
-                        "standard"
+                        "standard",
+                        "line",
+                        "paddle",
+                        "linear",
+                        "gitlab",
+                        "zoom",
+                        "twitch",
+                        "twilio"
                       ]
                     },
                     {
@@ -304,6 +639,10 @@
                 "invalidSignature": {
                   "type": "boolean",
                   "description": "Signed with a different secret"
+                },
+                "url": {
+                  "type": "string",
+                  "description": "The URL the app validates against, for providers that sign it (twilio). Default: the app's address and the request path"
                 }
               }
             },
@@ -344,6 +683,17 @@
                   "type": "integer",
                   "minimum": 0,
                   "description": "At most this many SQL statements for this request, not counting BEGIN/COMMIT (needs db.queries)"
+                },
+                "schema": {
+                  "description": "A JSON Schema the response JSON must match: inline, or a file relative to this one with an optional pointer, e.g. ../openapi.yaml#/components/schemas/Poll",
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object"
+                    }
+                  ]
                 }
               }
             },
@@ -353,7 +703,105 @@
               "additionalProperties": {
                 "type": "string"
               }
+            },
+            "graphql": {
+              "description": "Send a GraphQL operation as the JSON body: the query, or { query, variables, operationName }. A response with `errors` fails the step unless `expect.json` names `errors`",
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "object",
+                  "required": [
+                    "query"
+                  ],
+                  "additionalProperties": false,
+                  "properties": {
+                    "query": {
+                      "type": "string"
+                    },
+                    "variables": {
+                      "type": "object"
+                    },
+                    "operationName": {
+                      "type": "string"
+                    }
+                  }
+                }
+              ]
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Repeat the request until `expect` passes, for up to this many milliseconds (polling a job the app finishes later)"
+            },
+            "every": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "With `within`: milliseconds between attempts. Default 200"
+            },
+            "timeout": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Fail when the app hasn't answered within this many milliseconds (overrides http.timeout)"
             }
+          },
+          "not": {
+            "anyOf": [
+              {
+                "required": [
+                  "graphql",
+                  "json"
+                ]
+              },
+              {
+                "required": [
+                  "graphql",
+                  "form"
+                ]
+              },
+              {
+                "required": [
+                  "graphql",
+                  "body"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "json"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "form"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "body"
+                ]
+              },
+              {
+                "required": [
+                  "multipart",
+                  "graphql"
+                ]
+              },
+              {
+                "required": [
+                  "within",
+                  "concurrency"
+                ]
+              }
+            ]
+          },
+          "dependentRequired": {
+            "every": [
+              "within"
+            ]
           }
         },
         {
@@ -388,7 +836,7 @@
             },
             "fields": {
               "type": "object",
-              "description": "Values typed into the form by field name; true / false / a list of values check checkboxes and radios",
+              "description": "Values typed into the form by field name; true / false / a list of values check checkboxes and radios; { file: path } uploads a file through a file input",
               "additionalProperties": {
                 "oneOf": [
                   {
@@ -406,6 +854,36 @@
                         "number"
                       ]
                     }
+                  },
+                  {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                      "file": {
+                        "type": "string",
+                        "description": "Path relative to the scenario file"
+                      },
+                      "content": {},
+                      "filename": {
+                        "type": "string"
+                      },
+                      "type": {
+                        "type": "string"
+                      }
+                    },
+                    "oneOf": [
+                      {
+                        "required": [
+                          "file"
+                        ]
+                      },
+                      {
+                        "required": [
+                          "content"
+                        ]
+                      }
+                    ],
+                    "description": "A file for a file input"
                   }
                 ]
               }
@@ -432,7 +910,18 @@
                 "json": {
                   "description": "Subset of the response JSON"
                 },
-                "text": {}
+                "text": {},
+                "schema": {
+                  "description": "A JSON Schema the response JSON must match: inline, or a file relative to this one with an optional pointer, e.g. ../openapi.yaml#/components/schemas/Poll",
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object"
+                    }
+                  ]
+                }
               }
             },
             "capture": {
@@ -707,7 +1196,14 @@
                 "json": {
                   "description": "Subset of the request JSON"
                 },
-                "body": {}
+                "form": {
+                  "type": "object",
+                  "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                },
+                "body": {},
+                "variables": {
+                  "description": "Subset of the GraphQL variables (with `graphql`)"
+                }
               }
             },
             "times": {
@@ -719,7 +1215,17 @@
               "type": "integer",
               "minimum": 1,
               "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            },
+            "graphql": {
+              "type": "string",
+              "description": "A GraphQL operation (its operationName, or the name in the document), instead of METHOD /path. Matches POST and GET GraphQL requests to any path"
             }
+          },
+          "not": {
+            "required": [
+              "call",
+              "graphql"
+            ]
           }
         },
         {
@@ -792,6 +1298,105 @@
               "type": "integer",
               "minimum": 1,
               "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            },
+            "ignore": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Columns or tables to leave out: updated_at (in every table), orders.synced_at, sessions.*"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "set"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "set": {
+              "type": "object",
+              "minProperties": 1,
+              "propertyNames": {
+                "pattern": "^[A-Za-z_]\\w*$"
+              },
+              "description": "Variables for later steps, e.g. { orderId: \"{{$uuid}}\", expires: \"{{$now+1d}}\" }"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "required": [
+            "order"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "order": {
+              "type": "array",
+              "minItems": 2,
+              "description": "Calls to stubs in the order they must have happened (others may come in between): \"<stub> METHOD /path\" or { stub, call, when }",
+              "items": {
+                "oneOf": [
+                  {
+                    "type": "string",
+                    "pattern": "^\\S+\\s+([A-Za-z]+|\\*)\\s+/\\S*$"
+                  },
+                  {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": [
+                      "stub",
+                      "call"
+                    ],
+                    "properties": {
+                      "stub": {
+                        "type": "string"
+                      },
+                      "call": {
+                        "type": "string",
+                        "pattern": "^([A-Za-z]+|\\*)\\s+/\\S*$"
+                      },
+                      "when": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                          "query": {
+                            "type": "object"
+                          },
+                          "headers": {
+                            "type": "object"
+                          },
+                          "json": {
+                            "description": "Subset of the request JSON"
+                          },
+                          "form": {
+                            "type": "object",
+                            "description": "Subset of a form-encoded body's fields; bracket keys are nested (metadata[order] → metadata: { order })"
+                          },
+                          "body": {},
+                          "variables": {
+                            "description": "Subset of the GraphQL variables (with `graphql`)"
+                          }
+                        }
+                      }
+                    }
+                  }
+                ]
+              }
+            },
+            "within": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "Retry this step for up to this many milliseconds, for effects the app applies asynchronously"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
             }
           }
         },
@@ -993,6 +1598,27 @@
               "description": "Label shown when this step fails"
             }
           }
+        },
+        {
+          "type": "object",
+          "required": [
+            "use"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "use": {
+              "type": "string",
+              "description": "Run the steps of a `define:` entry; what they capture is visible afterwards"
+            },
+            "name": {
+              "type": "string",
+              "description": "Label shown when this step fails"
+            },
+            "with": {
+              "type": "object",
+              "description": "Values for the definition's params, available as {{param}} inside it"
+            }
+          }
         }
       ]
     },