schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/cli/__init__.py +3 -3
- schemathesis/cli/commands/run/__init__.py +148 -94
- schemathesis/cli/commands/run/context.py +72 -2
- schemathesis/cli/commands/run/executor.py +32 -12
- schemathesis/cli/commands/run/filters.py +1 -0
- schemathesis/cli/commands/run/handlers/cassettes.py +27 -46
- schemathesis/cli/commands/run/handlers/junitxml.py +1 -1
- schemathesis/cli/commands/run/handlers/output.py +72 -16
- schemathesis/cli/commands/run/hypothesis.py +30 -19
- schemathesis/cli/commands/run/reports.py +72 -0
- schemathesis/cli/commands/run/validation.py +18 -12
- schemathesis/cli/ext/groups.py +42 -13
- schemathesis/cli/ext/options.py +15 -8
- schemathesis/core/errors.py +79 -11
- schemathesis/core/failures.py +2 -1
- schemathesis/core/transforms.py +1 -1
- schemathesis/engine/errors.py +8 -3
- schemathesis/engine/phases/stateful/_executor.py +1 -1
- schemathesis/engine/phases/unit/__init__.py +2 -3
- schemathesis/engine/phases/unit/_executor.py +16 -13
- schemathesis/errors.py +6 -2
- schemathesis/filters.py +8 -0
- schemathesis/generation/coverage.py +6 -1
- schemathesis/generation/stateful/state_machine.py +49 -3
- schemathesis/pytest/lazy.py +2 -3
- schemathesis/pytest/plugin.py +2 -3
- schemathesis/schemas.py +1 -1
- schemathesis/specs/openapi/checks.py +27 -10
- schemathesis/specs/openapi/expressions/__init__.py +22 -6
- schemathesis/specs/openapi/expressions/nodes.py +15 -21
- schemathesis/specs/openapi/expressions/parser.py +1 -1
- schemathesis/specs/openapi/parameters.py +0 -2
- schemathesis/specs/openapi/schemas.py +13 -13
- schemathesis/specs/openapi/stateful/__init__.py +96 -23
- schemathesis/specs/openapi/{links.py → stateful/links.py} +60 -16
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/METADATA +1 -1
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/RECORD +40 -39
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/licenses/LICENSE +0 -0
@@ -13,7 +13,7 @@ from urllib.parse import urlparse
|
|
13
13
|
import click
|
14
14
|
|
15
15
|
from schemathesis import errors, experimental
|
16
|
-
from schemathesis.cli.commands.run.
|
16
|
+
from schemathesis.cli.commands.run.reports import ReportFormat
|
17
17
|
from schemathesis.cli.constants import DEFAULT_WORKERS
|
18
18
|
from schemathesis.core import rate_limit, string_to_boolean
|
19
19
|
from schemathesis.core.fs import file_exists
|
@@ -24,8 +24,10 @@ from schemathesis.generation.overrides import Override
|
|
24
24
|
INVALID_DERANDOMIZE_MESSAGE = (
|
25
25
|
"`--generation-deterministic` implies no database, so passing `--generation-database` too is invalid."
|
26
26
|
)
|
27
|
-
|
28
|
-
"
|
27
|
+
INVALID_REPORT_USAGE = (
|
28
|
+
"Can't use `--report-preserve-bytes` without enabling cassette formats. "
|
29
|
+
"Enable VCR or HAR format with `--report=vcr`, `--report-vcr-path`, "
|
30
|
+
"`--report=har`, or `--report-har-path`"
|
29
31
|
)
|
30
32
|
INVALID_SCHEMA_MESSAGE = "Invalid SCHEMA, must be a valid URL or file path."
|
31
33
|
FILE_DOES_NOT_EXIST_MESSAGE = "The specified file does not exist. Please provide a valid path to an existing file."
|
@@ -33,7 +35,7 @@ INVALID_BASE_URL_MESSAGE = (
|
|
33
35
|
"The provided base URL is invalid. This URL serves as a prefix for all API endpoints you want to test. "
|
34
36
|
"Make sure it is a properly formatted URL."
|
35
37
|
)
|
36
|
-
MISSING_BASE_URL_MESSAGE = "The `--
|
38
|
+
MISSING_BASE_URL_MESSAGE = "The `--url` option is required when specifying a schema via a file."
|
37
39
|
MISSING_REQUEST_CERT_MESSAGE = "The `--request-cert` option must be specified if `--request-cert-key` is used."
|
38
40
|
|
39
41
|
|
@@ -240,10 +242,18 @@ def validate_request_cert_key(
|
|
240
242
|
return raw_value
|
241
243
|
|
242
244
|
|
243
|
-
def
|
244
|
-
if raw_value
|
245
|
-
|
246
|
-
|
245
|
+
def validate_preserve_bytes(ctx: click.core.Context, param: click.core.Parameter, raw_value: bool) -> bool:
|
246
|
+
if not raw_value:
|
247
|
+
return False
|
248
|
+
|
249
|
+
report_formats = ctx.params.get("report_formats", []) or []
|
250
|
+
vcr_enabled = ReportFormat.VCR in report_formats or ctx.params.get("report_vcr_path")
|
251
|
+
har_enabled = ReportFormat.HAR in report_formats or ctx.params.get("report_har_path")
|
252
|
+
|
253
|
+
if not (vcr_enabled or har_enabled):
|
254
|
+
raise click.UsageError(INVALID_REPORT_USAGE)
|
255
|
+
|
256
|
+
return True
|
247
257
|
|
248
258
|
|
249
259
|
def convert_experimental(
|
@@ -302,10 +312,6 @@ def convert_status_codes(
|
|
302
312
|
return value
|
303
313
|
|
304
314
|
|
305
|
-
def convert_cassette_format(ctx: click.core.Context, param: click.core.Parameter, value: str) -> CassetteFormat:
|
306
|
-
return CassetteFormat.from_str(value)
|
307
|
-
|
308
|
-
|
309
315
|
def convert_generation_mode(ctx: click.core.Context, param: click.core.Parameter, value: str) -> list[GenerationMode]:
|
310
316
|
if value == "all":
|
311
317
|
return GenerationMode.all()
|
schemathesis/cli/ext/groups.py
CHANGED
@@ -1,34 +1,59 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
|
3
|
+
import textwrap
|
4
4
|
from typing import Any, Callable
|
5
5
|
|
6
6
|
import click
|
7
7
|
|
8
|
-
GROUPS:
|
8
|
+
GROUPS: dict[str, OptionGroup] = {}
|
9
|
+
|
10
|
+
|
11
|
+
class OptionGroup:
|
12
|
+
__slots__ = ("order", "name", "description", "options")
|
13
|
+
|
14
|
+
def __init__(
|
15
|
+
self,
|
16
|
+
name: str,
|
17
|
+
*,
|
18
|
+
order: int | None = None,
|
19
|
+
description: str | None = None,
|
20
|
+
):
|
21
|
+
self.name = name
|
22
|
+
self.description = description
|
23
|
+
self.order = order if order is not None else len(GROUPS) * 100
|
24
|
+
self.options: list[tuple[str, str]] = []
|
9
25
|
|
10
26
|
|
11
27
|
class CommandWithGroupedOptions(click.Command):
|
12
28
|
def format_options(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
13
|
-
groups
|
29
|
+
# Collect options into groups or ungrouped list
|
14
30
|
for param in self.get_params(ctx):
|
15
31
|
rv = param.get_help_record(ctx)
|
16
32
|
if rv is not None:
|
17
|
-
|
33
|
+
option_repr, message = rv
|
18
34
|
if isinstance(param.type, click.Choice):
|
19
35
|
message += (
|
20
36
|
getattr(param.type, "choices_repr", None)
|
21
37
|
or f" [possible values: {', '.join(param.type.choices)}]"
|
22
38
|
)
|
23
39
|
|
24
|
-
if isinstance(param, GroupedOption):
|
25
|
-
group = param.group
|
40
|
+
if isinstance(param, GroupedOption) and param.group is not None:
|
41
|
+
group = GROUPS.get(param.group)
|
42
|
+
if group:
|
43
|
+
group.options.append((option_repr, message))
|
26
44
|
else:
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
45
|
+
GROUPS["Global options"].options.append((option_repr, message))
|
46
|
+
|
47
|
+
groups = sorted(GROUPS.values(), key=lambda g: g.order)
|
48
|
+
# Format each group
|
49
|
+
for group in groups:
|
50
|
+
with formatter.section(group.name):
|
51
|
+
if group.description:
|
52
|
+
formatter.write(textwrap.indent(group.description, " " * formatter.current_indent))
|
53
|
+
formatter.write("\n\n")
|
54
|
+
|
55
|
+
if group.options:
|
56
|
+
formatter.write_dl(group.options)
|
32
57
|
|
33
58
|
|
34
59
|
class GroupedOption(click.Option):
|
@@ -37,8 +62,12 @@ class GroupedOption(click.Option):
|
|
37
62
|
self.group = group
|
38
63
|
|
39
64
|
|
40
|
-
def group(
|
41
|
-
|
65
|
+
def group(
|
66
|
+
name: str,
|
67
|
+
*,
|
68
|
+
description: str | None = None,
|
69
|
+
) -> Callable:
|
70
|
+
GROUPS[name] = OptionGroup(name, description=description)
|
42
71
|
|
43
72
|
def _inner(cmd: Callable) -> Callable:
|
44
73
|
for param in reversed(cmd.__click_params__): # type: ignore[attr-defined]
|
schemathesis/cli/ext/options.py
CHANGED
@@ -22,8 +22,13 @@ class CustomHelpMessageChoice(click.Choice):
|
|
22
22
|
|
23
23
|
class BaseCsvChoice(click.Choice):
|
24
24
|
def parse_value(self, value: str) -> tuple[list[str], set[str]]:
|
25
|
-
selected = [item for item in value.split(",") if item]
|
26
|
-
|
25
|
+
selected = [item.strip() for item in value.split(",") if item.strip()]
|
26
|
+
if not self.case_sensitive:
|
27
|
+
invalid_options = {
|
28
|
+
item for item in selected if item.upper() not in {choice.upper() for choice in self.choices}
|
29
|
+
}
|
30
|
+
else:
|
31
|
+
invalid_options = set(selected) - set(self.choices)
|
27
32
|
return selected, invalid_options
|
28
33
|
|
29
34
|
def fail_on_invalid_options(self, invalid_options: set[str], selected: list[str]) -> NoReturn: # type: ignore[misc]
|
@@ -44,16 +49,18 @@ class CsvChoice(BaseCsvChoice):
|
|
44
49
|
|
45
50
|
|
46
51
|
class CsvEnumChoice(BaseCsvChoice):
|
47
|
-
def __init__(self, choices: type[Enum]):
|
52
|
+
def __init__(self, choices: type[Enum], case_sensitive: bool = False):
|
48
53
|
self.enum = choices
|
49
|
-
super().__init__(tuple(el.name for el in choices))
|
54
|
+
super().__init__(tuple(el.name.lower() for el in choices), case_sensitive=case_sensitive)
|
50
55
|
|
51
|
-
def convert(
|
52
|
-
self, value: str, param: click.core.Parameter | None, ctx: click.core.Context | None
|
53
|
-
) -> list[Enum]:
|
56
|
+
def convert(self, value: str, param: click.core.Parameter | None, ctx: click.core.Context | None) -> list[Enum]:
|
54
57
|
selected, invalid_options = self.parse_value(value)
|
55
58
|
if not invalid_options and selected:
|
56
|
-
|
59
|
+
# Match case-insensitively to find the correct enum
|
60
|
+
return [
|
61
|
+
next(enum_value for enum_value in self.enum if enum_value.value.upper() == item.upper())
|
62
|
+
for item in selected
|
63
|
+
]
|
57
64
|
self.fail_on_invalid_options(invalid_options, selected)
|
58
65
|
|
59
66
|
|
schemathesis/core/errors.py
CHANGED
@@ -35,20 +35,16 @@ class InvalidSchema(SchemathesisError):
|
|
35
35
|
|
36
36
|
def __init__(
|
37
37
|
self,
|
38
|
-
message: str
|
38
|
+
message: str,
|
39
39
|
path: str | None = None,
|
40
40
|
method: str | None = None,
|
41
|
-
full_path: str | None = None,
|
42
41
|
) -> None:
|
43
42
|
self.message = message
|
44
43
|
self.path = path
|
45
44
|
self.method = method
|
46
|
-
self.full_path = full_path
|
47
45
|
|
48
46
|
@classmethod
|
49
|
-
def from_jsonschema_error(
|
50
|
-
cls, error: ValidationError, path: str | None, method: str | None, full_path: str | None
|
51
|
-
) -> InvalidSchema:
|
47
|
+
def from_jsonschema_error(cls, error: ValidationError, path: str | None, method: str | None) -> InvalidSchema:
|
52
48
|
if error.absolute_path:
|
53
49
|
part = error.absolute_path[-1]
|
54
50
|
if isinstance(part, int) and len(error.absolute_path) > 1:
|
@@ -69,11 +65,11 @@ class InvalidSchema(SchemathesisError):
|
|
69
65
|
else:
|
70
66
|
message += error.message
|
71
67
|
message += f"\n\n{SCHEMA_ERROR_SUGGESTION}"
|
72
|
-
return cls(message, path=path, method=method
|
68
|
+
return cls(message, path=path, method=method)
|
73
69
|
|
74
70
|
@classmethod
|
75
71
|
def from_reference_resolution_error(
|
76
|
-
cls, error: RefResolutionError, path: str | None, method: str | None
|
72
|
+
cls, error: RefResolutionError, path: str | None, method: str | None
|
77
73
|
) -> InvalidSchema:
|
78
74
|
notes = getattr(error, "__notes__", [])
|
79
75
|
# Some exceptions don't have the actual reference in them, hence we add it manually via notes
|
@@ -83,7 +79,7 @@ class InvalidSchema(SchemathesisError):
|
|
83
79
|
message += f"\n\nError details:\n JSON pointer: {pointer}"
|
84
80
|
message += "\n This typically means that the schema is referencing a component that doesn't exist."
|
85
81
|
message += f"\n\n{SCHEMA_ERROR_SUGGESTION}"
|
86
|
-
return cls(message, path=path, method=method
|
82
|
+
return cls(message, path=path, method=method)
|
87
83
|
|
88
84
|
def as_failing_test_function(self) -> Callable:
|
89
85
|
"""Create a test function that will fail.
|
@@ -102,8 +98,80 @@ class InvalidRegexType(InvalidSchema):
|
|
102
98
|
"""Raised when an invalid type is used where a regex pattern is expected."""
|
103
99
|
|
104
100
|
|
105
|
-
class
|
106
|
-
"""
|
101
|
+
class InvalidStateMachine(SchemathesisError):
|
102
|
+
"""Collection of validation errors found in API state machine transitions.
|
103
|
+
|
104
|
+
Raised during schema initialization when one or more transitions
|
105
|
+
contain invalid definitions, such as references to non-existent parameters
|
106
|
+
or operations.
|
107
|
+
"""
|
108
|
+
|
109
|
+
errors: list[InvalidTransition]
|
110
|
+
|
111
|
+
__slots__ = ("errors",)
|
112
|
+
|
113
|
+
def __init__(self, errors: list[InvalidTransition]) -> None:
|
114
|
+
self.errors = errors
|
115
|
+
|
116
|
+
def __str__(self) -> str:
|
117
|
+
"""Format state machine validation errors in a clear, hierarchical structure."""
|
118
|
+
result = "The following API operations contain invalid link definitions:"
|
119
|
+
|
120
|
+
# Group transitions by source operation, then by target and status
|
121
|
+
by_source: dict[str, dict[tuple[str, str], list[InvalidTransition]]] = {}
|
122
|
+
for transition in self.errors:
|
123
|
+
source_group = by_source.setdefault(transition.source, {})
|
124
|
+
target_key = (transition.target, transition.status_code)
|
125
|
+
source_group.setdefault(target_key, []).append(transition)
|
126
|
+
|
127
|
+
for source, target_groups in by_source.items():
|
128
|
+
for (target, status), transitions in target_groups.items():
|
129
|
+
for transition in transitions:
|
130
|
+
result += f"\n\n {_format_transition(source, status, transition.name, target)}\n"
|
131
|
+
for error in transition.errors:
|
132
|
+
result += f"\n - {error.message}"
|
133
|
+
return result
|
134
|
+
|
135
|
+
|
136
|
+
def _format_transition(source: str, status: str, transition: str, target: str) -> str:
|
137
|
+
return f"{source} -> [{status}] {transition} -> {target}"
|
138
|
+
|
139
|
+
|
140
|
+
class InvalidTransition(SchemathesisError):
|
141
|
+
"""Raised when a stateful transition contains one or more errors."""
|
142
|
+
|
143
|
+
name: str
|
144
|
+
source: str
|
145
|
+
target: str
|
146
|
+
status_code: str
|
147
|
+
errors: list[TransitionValidationError]
|
148
|
+
|
149
|
+
__slots__ = ("name", "source", "target", "status_code", "errors")
|
150
|
+
|
151
|
+
def __init__(
|
152
|
+
self,
|
153
|
+
name: str,
|
154
|
+
source: str,
|
155
|
+
target: str,
|
156
|
+
status_code: str,
|
157
|
+
errors: list[TransitionValidationError],
|
158
|
+
) -> None:
|
159
|
+
self.name = name
|
160
|
+
self.source = source
|
161
|
+
self.target = target
|
162
|
+
self.status_code = status_code
|
163
|
+
self.errors = errors
|
164
|
+
|
165
|
+
|
166
|
+
class TransitionValidationError(SchemathesisError):
|
167
|
+
"""Single validation error found during stateful transition validation."""
|
168
|
+
|
169
|
+
message: str
|
170
|
+
|
171
|
+
__slots__ = ("message",)
|
172
|
+
|
173
|
+
def __init__(self, message: str) -> None:
|
174
|
+
self.message = message
|
107
175
|
|
108
176
|
|
109
177
|
class MalformedMediaType(ValueError):
|
schemathesis/core/failures.py
CHANGED
@@ -228,9 +228,10 @@ class MalformedJson(Failure):
|
|
228
228
|
|
229
229
|
@classmethod
|
230
230
|
def from_exception(cls, *, operation: str, exc: JSONDecodeError) -> MalformedJson:
|
231
|
+
message = f"Response must be valid JSON with 'Content-Type: application/json' header:\n\n {exc}"
|
231
232
|
return cls(
|
232
233
|
operation=operation,
|
233
|
-
message=
|
234
|
+
message=message,
|
234
235
|
validation_message=exc.msg,
|
235
236
|
document=exc.doc,
|
236
237
|
position=exc.pos,
|
schemathesis/core/transforms.py
CHANGED
@@ -106,7 +106,7 @@ def resolve_pointer(document: Any, pointer: str) -> dict | list | str | int | fl
|
|
106
106
|
elif isinstance(target, list):
|
107
107
|
try:
|
108
108
|
target = target[int(token)]
|
109
|
-
except IndexError:
|
109
|
+
except (IndexError, ValueError):
|
110
110
|
return UNRESOLVABLE
|
111
111
|
else:
|
112
112
|
return UNRESOLVABLE
|
schemathesis/engine/errors.py
CHANGED
@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Callable, Iterator, Sequence, cast
|
|
14
14
|
from schemathesis import errors
|
15
15
|
from schemathesis.core.errors import (
|
16
16
|
RECURSIVE_REFERENCE_ERROR_MESSAGE,
|
17
|
-
|
17
|
+
InvalidTransition,
|
18
18
|
SerializationNotPossible,
|
19
19
|
format_exception,
|
20
20
|
get_request_error_extras,
|
@@ -77,7 +77,7 @@ class EngineErrorInfo:
|
|
77
77
|
"""A general error description."""
|
78
78
|
import requests
|
79
79
|
|
80
|
-
if isinstance(self._error,
|
80
|
+
if isinstance(self._error, InvalidTransition):
|
81
81
|
return "Invalid Link Definition"
|
82
82
|
|
83
83
|
if isinstance(self._error, requests.RequestException):
|
@@ -101,6 +101,7 @@ class EngineErrorInfo:
|
|
101
101
|
return {
|
102
102
|
RuntimeErrorKind.SCHEMA_UNSUPPORTED: "Unsupported Schema",
|
103
103
|
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Missing Open API links",
|
104
|
+
RuntimeErrorKind.SCHEMA_INVALID_STATE_MACHINE: "Invalid OpenAPI Links Definition",
|
104
105
|
RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Unknown GraphQL Scalar",
|
105
106
|
RuntimeErrorKind.SERIALIZATION_UNBOUNDED_PREFIX: "XML serialization error",
|
106
107
|
RuntimeErrorKind.SERIALIZATION_NOT_POSSIBLE: "Serialization not possible",
|
@@ -169,6 +170,7 @@ class EngineErrorInfo:
|
|
169
170
|
def has_useful_traceback(self) -> bool:
|
170
171
|
return self._kind not in (
|
171
172
|
RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION,
|
173
|
+
RuntimeErrorKind.SCHEMA_INVALID_STATE_MACHINE,
|
172
174
|
RuntimeErrorKind.SCHEMA_UNSUPPORTED,
|
173
175
|
RuntimeErrorKind.SCHEMA_GENERIC,
|
174
176
|
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND,
|
@@ -246,7 +248,7 @@ def get_runtime_error_suggestion(error_type: RuntimeErrorKind, bold: Callable[[s
|
|
246
248
|
return f"Bypass this health check using {bold(f'`--suppress-health-check={label}`')}."
|
247
249
|
|
248
250
|
return {
|
249
|
-
RuntimeErrorKind.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--
|
251
|
+
RuntimeErrorKind.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--tls-verify=false`')}.",
|
250
252
|
RuntimeErrorKind.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.",
|
251
253
|
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Review your endpoint filters to include linked operations",
|
252
254
|
RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax.\n"
|
@@ -302,6 +304,7 @@ class RuntimeErrorKind(str, enum.Enum):
|
|
302
304
|
HYPOTHESIS_HEALTH_CHECK_LARGE_BASE_EXAMPLE = "hypothesis_health_check_large_base_example"
|
303
305
|
|
304
306
|
SCHEMA_INVALID_REGULAR_EXPRESSION = "schema_invalid_regular_expression"
|
307
|
+
SCHEMA_INVALID_STATE_MACHINE = "schema_invalid_state_machine"
|
305
308
|
SCHEMA_NO_LINKS_FOUND = "schema_no_links_found"
|
306
309
|
SCHEMA_UNSUPPORTED = "schema_unsupported"
|
307
310
|
SCHEMA_GENERIC = "schema_generic"
|
@@ -354,6 +357,8 @@ def _classify(*, error: Exception) -> RuntimeErrorKind:
|
|
354
357
|
if isinstance(error, errors.InvalidRegexPattern):
|
355
358
|
return RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION
|
356
359
|
return RuntimeErrorKind.SCHEMA_GENERIC
|
360
|
+
if isinstance(error, errors.InvalidStateMachine):
|
361
|
+
return RuntimeErrorKind.SCHEMA_INVALID_STATE_MACHINE
|
357
362
|
if isinstance(error, errors.NoLinksFound):
|
358
363
|
return RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND
|
359
364
|
if isinstance(error, UnsupportedRecursiveReference):
|
@@ -134,7 +134,7 @@ def execute_state_machine_loop(
|
|
134
134
|
return result
|
135
135
|
|
136
136
|
def validate_response(
|
137
|
-
self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
|
137
|
+
self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = (), **kwargs: Any
|
138
138
|
) -> None:
|
139
139
|
self.recorder.record_response(case_id=case.id, response=response)
|
140
140
|
ctx.collect_metric(case, response)
|
@@ -123,7 +123,7 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
|
|
123
123
|
else:
|
124
124
|
error = result.err()
|
125
125
|
if error.method:
|
126
|
-
label = f"{error.method.upper()} {error.
|
126
|
+
label = f"{error.method.upper()} {error.path}"
|
127
127
|
scenario_started = events.ScenarioStarted(
|
128
128
|
label=label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id
|
129
129
|
)
|
@@ -149,12 +149,11 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
|
|
149
149
|
)
|
150
150
|
)
|
151
151
|
else:
|
152
|
-
assert error.full_path is not None
|
153
152
|
events_queue.put(
|
154
153
|
events.NonFatalError(
|
155
154
|
error=error,
|
156
155
|
phase=PhaseName.UNIT_TESTING,
|
157
|
-
label=error.
|
156
|
+
label=error.path,
|
158
157
|
related_to_operation=False,
|
159
158
|
)
|
160
159
|
)
|
@@ -70,6 +70,19 @@ def run_test(
|
|
70
70
|
error=error, phase=PhaseName.UNIT_TESTING, label=operation.label, related_to_operation=True
|
71
71
|
)
|
72
72
|
|
73
|
+
def scenario_finished(status: Status) -> events.ScenarioFinished:
|
74
|
+
return events.ScenarioFinished(
|
75
|
+
id=scenario_started.id,
|
76
|
+
suite_id=suite_id,
|
77
|
+
phase=PhaseName.UNIT_TESTING,
|
78
|
+
label=operation.label,
|
79
|
+
recorder=recorder,
|
80
|
+
status=status,
|
81
|
+
elapsed_time=time.monotonic() - test_start_time,
|
82
|
+
skip_reason=skip_reason,
|
83
|
+
is_final=False,
|
84
|
+
)
|
85
|
+
|
73
86
|
try:
|
74
87
|
setup_hypothesis_database_key(test_function, operation)
|
75
88
|
with catch_warnings(record=True) as warnings, ignore_hypothesis_output():
|
@@ -111,6 +124,7 @@ def run_test(
|
|
111
124
|
status = Status.ERROR
|
112
125
|
yield non_fatal_error(hypothesis.errors.Unsatisfiable("Failed to generate test cases for this API operation"))
|
113
126
|
except KeyboardInterrupt:
|
127
|
+
yield scenario_finished(Status.INTERRUPTED)
|
114
128
|
yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
|
115
129
|
return
|
116
130
|
except AssertionError as exc: # May come from `hypothesis-jsonschema` or `hypothesis`
|
@@ -131,7 +145,6 @@ def run_test(
|
|
131
145
|
exc,
|
132
146
|
path=operation.path,
|
133
147
|
method=operation.method,
|
134
|
-
full_path=operation.schema.get_full_path(operation.path),
|
135
148
|
)
|
136
149
|
)
|
137
150
|
except HypothesisRefResolutionError:
|
@@ -194,20 +207,10 @@ def run_test(
|
|
194
207
|
if invalid_headers:
|
195
208
|
status = Status.ERROR
|
196
209
|
yield non_fatal_error(InvalidHeadersExample.from_headers(invalid_headers))
|
197
|
-
test_elapsed_time = time.monotonic() - test_start_time
|
198
210
|
for error in deduplicate_errors(errors):
|
199
211
|
yield non_fatal_error(error)
|
200
|
-
|
201
|
-
|
202
|
-
suite_id=suite_id,
|
203
|
-
phase=PhaseName.UNIT_TESTING,
|
204
|
-
label=operation.label,
|
205
|
-
recorder=recorder,
|
206
|
-
status=status,
|
207
|
-
elapsed_time=test_elapsed_time,
|
208
|
-
skip_reason=skip_reason,
|
209
|
-
is_final=False,
|
210
|
-
)
|
212
|
+
|
213
|
+
yield scenario_finished(status)
|
211
214
|
|
212
215
|
|
213
216
|
def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
|
schemathesis/errors.py
CHANGED
@@ -4,17 +4,19 @@ from schemathesis.core.errors import (
|
|
4
4
|
IncorrectUsage,
|
5
5
|
InternalError,
|
6
6
|
InvalidHeadersExample,
|
7
|
-
InvalidLinkDefinition,
|
8
7
|
InvalidRateLimit,
|
9
8
|
InvalidRegexPattern,
|
10
9
|
InvalidRegexType,
|
11
10
|
InvalidSchema,
|
11
|
+
InvalidStateMachine,
|
12
|
+
InvalidTransition,
|
12
13
|
LoaderError,
|
13
14
|
NoLinksFound,
|
14
15
|
OperationNotFound,
|
15
16
|
SchemathesisError,
|
16
17
|
SerializationError,
|
17
18
|
SerializationNotPossible,
|
19
|
+
TransitionValidationError,
|
18
20
|
UnboundPrefix,
|
19
21
|
)
|
20
22
|
|
@@ -22,16 +24,18 @@ __all__ = [
|
|
22
24
|
"IncorrectUsage",
|
23
25
|
"InternalError",
|
24
26
|
"InvalidHeadersExample",
|
25
|
-
"InvalidLinkDefinition",
|
26
27
|
"InvalidRateLimit",
|
27
28
|
"InvalidRegexPattern",
|
28
29
|
"InvalidRegexType",
|
29
30
|
"InvalidSchema",
|
31
|
+
"InvalidStateMachine",
|
32
|
+
"InvalidTransition",
|
30
33
|
"LoaderError",
|
31
34
|
"OperationNotFound",
|
32
35
|
"NoLinksFound",
|
33
36
|
"SchemathesisError",
|
34
37
|
"SerializationError",
|
35
38
|
"SerializationNotPossible",
|
39
|
+
"TransitionValidationError",
|
36
40
|
"UnboundPrefix",
|
37
41
|
]
|
schemathesis/filters.py
CHANGED
@@ -268,6 +268,8 @@ class FilterSet:
|
|
268
268
|
# To match anything the regex should match the expected value, hence passing them together is useless
|
269
269
|
raise IncorrectUsage(ERROR_EXPECTED_AND_REGEX)
|
270
270
|
if expected is not None:
|
271
|
+
if attribute == "method":
|
272
|
+
expected = _normalize_method(expected)
|
271
273
|
matchers.append(Matcher.for_value(attribute, expected))
|
272
274
|
if regex is not None:
|
273
275
|
matchers.append(Matcher.for_regex(attribute, regex))
|
@@ -283,6 +285,12 @@ class FilterSet:
|
|
283
285
|
self._excludes.add(filter_)
|
284
286
|
|
285
287
|
|
288
|
+
def _normalize_method(value: FilterValue) -> FilterValue:
|
289
|
+
if isinstance(value, list):
|
290
|
+
return [item.upper() for item in value]
|
291
|
+
return value.upper()
|
292
|
+
|
293
|
+
|
286
294
|
def attach_filter_chain(
|
287
295
|
target: Callable,
|
288
296
|
attribute: str,
|
@@ -437,7 +437,12 @@ def cover_schema_iter(
|
|
437
437
|
elif key == "required":
|
438
438
|
template = template or ctx.generate_from_schema(_get_template_schema(schema, "object"))
|
439
439
|
yield from _negative_required(ctx, template, value)
|
440
|
-
elif
|
440
|
+
elif (
|
441
|
+
key == "additionalProperties"
|
442
|
+
and not value
|
443
|
+
and "pattern" not in schema
|
444
|
+
and schema.get("type") in ["object", None]
|
445
|
+
):
|
441
446
|
template = template or ctx.generate_from_schema(_get_template_schema(schema, "object"))
|
442
447
|
yield NegativeValue(
|
443
448
|
{**template, UNKNOWN_PROPERTY_KEY: UNKNOWN_PROPERTY_VALUE},
|
@@ -68,6 +68,52 @@ class ExtractedParam:
|
|
68
68
|
__slots__ = ("definition", "value")
|
69
69
|
|
70
70
|
|
71
|
+
@dataclass
|
72
|
+
class ExtractionFailure:
|
73
|
+
"""Represents a failure to extract data from a transition."""
|
74
|
+
|
75
|
+
# e.g., "GetUser"
|
76
|
+
id: str
|
77
|
+
case_id: str
|
78
|
+
# e.g., "POST /users"
|
79
|
+
source: str
|
80
|
+
# e.g., "GET /users/{userId}"
|
81
|
+
target: str
|
82
|
+
# e.g., "userId"
|
83
|
+
parameter_name: str
|
84
|
+
# e.g., "$response.body#/id"
|
85
|
+
expression: str
|
86
|
+
# Previous test cases in the chain, from newest to oldest
|
87
|
+
# Stored as a case + response pair
|
88
|
+
history: list[tuple[Case, Response]]
|
89
|
+
# The actual response that caused the failure
|
90
|
+
response: Response
|
91
|
+
error: Exception | None
|
92
|
+
|
93
|
+
__slots__ = ("id", "case_id", "source", "target", "parameter_name", "expression", "history", "response", "error")
|
94
|
+
|
95
|
+
def __eq__(self, other: object) -> bool:
|
96
|
+
assert isinstance(other, ExtractionFailure)
|
97
|
+
return (
|
98
|
+
self.source == other.source
|
99
|
+
and self.target == other.target
|
100
|
+
and self.id == other.id
|
101
|
+
and self.parameter_name == other.parameter_name
|
102
|
+
and self.expression == other.expression
|
103
|
+
)
|
104
|
+
|
105
|
+
def __hash__(self) -> int:
|
106
|
+
return hash(
|
107
|
+
(
|
108
|
+
self.source,
|
109
|
+
self.target,
|
110
|
+
self.id,
|
111
|
+
self.parameter_name,
|
112
|
+
self.expression,
|
113
|
+
)
|
114
|
+
)
|
115
|
+
|
116
|
+
|
71
117
|
@dataclass
|
72
118
|
class StepOutput:
|
73
119
|
"""Output from a single transition of a state machine."""
|
@@ -172,7 +218,7 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
172
218
|
kwargs = self.get_call_kwargs(input.case)
|
173
219
|
response = self.call(input.case, **kwargs)
|
174
220
|
self.after_call(response, input.case)
|
175
|
-
self.validate_response(response, input.case)
|
221
|
+
self.validate_response(response, input.case, **kwargs)
|
176
222
|
return StepOutput(response, input.case)
|
177
223
|
|
178
224
|
def before_call(self, case: Case) -> None:
|
@@ -266,7 +312,7 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
266
312
|
return {}
|
267
313
|
|
268
314
|
def validate_response(
|
269
|
-
self, response: Response, case: Case, additional_checks: list[CheckFunction] | None = None
|
315
|
+
self, response: Response, case: Case, additional_checks: list[CheckFunction] | None = None, **kwargs: Any
|
270
316
|
) -> None:
|
271
317
|
"""Validate an API response.
|
272
318
|
|
@@ -298,4 +344,4 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
298
344
|
all provided checks rather than only the first encountered exception.
|
299
345
|
"""
|
300
346
|
__tracebackhide__ = True
|
301
|
-
case.validate_response(response, additional_checks=additional_checks)
|
347
|
+
case.validate_response(response, additional_checks=additional_checks, transport_kwargs=kwargs)
|
schemathesis/pytest/lazy.py
CHANGED
@@ -168,7 +168,7 @@ class LazySchema:
|
|
168
168
|
for result in tests:
|
169
169
|
if isinstance(result, Ok):
|
170
170
|
operation, sub_test = result.ok()
|
171
|
-
subtests.item._nodeid = f"{node_id}[{operation.method.upper()} {operation.
|
171
|
+
subtests.item._nodeid = f"{node_id}[{operation.method.upper()} {operation.path}]"
|
172
172
|
run_subtest(operation, fixtures, sub_test, subtests)
|
173
173
|
else:
|
174
174
|
_schema_error(subtests, result.err(), node_id)
|
@@ -236,8 +236,7 @@ SEPARATOR = "\n===================="
|
|
236
236
|
def _schema_error(subtests: SubTests, error: InvalidSchema, node_id: str) -> None:
|
237
237
|
"""Run a failing test, that will show the underlying problem."""
|
238
238
|
sub_test = error.as_failing_test_function()
|
239
|
-
|
240
|
-
kwargs = {"path": error.full_path}
|
239
|
+
kwargs = {"path": error.path}
|
241
240
|
if error.method:
|
242
241
|
kwargs["method"] = error.method.upper()
|
243
242
|
subtests.item._nodeid = _get_partial_node_name(node_id, **kwargs)
|