schemathesis 4.0.0a1__py3-none-any.whl → 4.0.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/checks.py +6 -4
- schemathesis/cli/__init__.py +12 -1
- schemathesis/cli/commands/run/__init__.py +4 -4
- schemathesis/cli/commands/run/events.py +19 -4
- schemathesis/cli/commands/run/executor.py +9 -3
- schemathesis/cli/commands/run/filters.py +27 -19
- schemathesis/cli/commands/run/handlers/base.py +1 -1
- schemathesis/cli/commands/run/handlers/cassettes.py +1 -3
- schemathesis/cli/commands/run/handlers/output.py +860 -201
- schemathesis/cli/commands/run/validation.py +1 -1
- schemathesis/cli/ext/options.py +4 -1
- schemathesis/core/errors.py +8 -0
- schemathesis/core/failures.py +54 -24
- schemathesis/engine/core.py +1 -1
- schemathesis/engine/errors.py +11 -5
- schemathesis/engine/events.py +3 -97
- schemathesis/engine/phases/stateful/__init__.py +2 -0
- schemathesis/engine/phases/stateful/_executor.py +22 -50
- schemathesis/engine/phases/unit/__init__.py +1 -0
- schemathesis/engine/phases/unit/_executor.py +2 -1
- schemathesis/engine/phases/unit/_pool.py +1 -1
- schemathesis/engine/recorder.py +29 -23
- schemathesis/errors.py +19 -13
- schemathesis/generation/coverage.py +4 -4
- schemathesis/generation/hypothesis/builder.py +15 -12
- schemathesis/generation/stateful/state_machine.py +61 -45
- schemathesis/graphql/checks.py +3 -9
- schemathesis/openapi/checks.py +8 -33
- schemathesis/schemas.py +34 -14
- schemathesis/specs/graphql/schemas.py +16 -15
- schemathesis/specs/openapi/checks.py +50 -27
- schemathesis/specs/openapi/expressions/__init__.py +11 -15
- schemathesis/specs/openapi/expressions/nodes.py +20 -20
- schemathesis/specs/openapi/links.py +139 -118
- schemathesis/specs/openapi/patterns.py +170 -2
- schemathesis/specs/openapi/schemas.py +60 -36
- schemathesis/specs/openapi/stateful/__init__.py +185 -113
- schemathesis/specs/openapi/stateful/control.py +87 -0
- {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/METADATA +1 -1
- {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/RECORD +43 -43
- schemathesis/specs/openapi/expressions/context.py +0 -14
- {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/licenses/LICENSE +0 -0
schemathesis/engine/recorder.py
CHANGED
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
import base64
|
4
4
|
import time
|
5
|
-
import uuid
|
6
5
|
from dataclasses import dataclass
|
7
6
|
from typing import TYPE_CHECKING, Iterator, cast
|
8
7
|
|
@@ -14,6 +13,8 @@ from schemathesis.generation.case import Case
|
|
14
13
|
if TYPE_CHECKING:
|
15
14
|
import requests
|
16
15
|
|
16
|
+
from schemathesis.generation.stateful.state_machine import Transition
|
17
|
+
|
17
18
|
|
18
19
|
@dataclass
|
19
20
|
class ScenarioRecorder:
|
@@ -22,7 +23,6 @@ class ScenarioRecorder:
|
|
22
23
|
Records test cases, their hierarchy, API interactions, and results of checks performed during execution.
|
23
24
|
"""
|
24
25
|
|
25
|
-
id: uuid.UUID
|
26
26
|
# Human-readable label
|
27
27
|
label: str
|
28
28
|
|
@@ -33,18 +33,17 @@ class ScenarioRecorder:
|
|
33
33
|
# Network interactions by test case ID
|
34
34
|
interactions: dict[str, Interaction]
|
35
35
|
|
36
|
-
__slots__ = ("
|
36
|
+
__slots__ = ("label", "status", "roots", "cases", "checks", "interactions")
|
37
37
|
|
38
38
|
def __init__(self, *, label: str) -> None:
|
39
|
-
self.id = uuid.uuid4()
|
40
39
|
self.label = label
|
41
40
|
self.cases = {}
|
42
41
|
self.checks = {}
|
43
42
|
self.interactions = {}
|
44
43
|
|
45
|
-
def record_case(self, *, parent_id: str | None, case: Case) -> None:
|
44
|
+
def record_case(self, *, parent_id: str | None, transition: Transition | None, case: Case) -> None:
|
46
45
|
"""Record a test case and its relationship to a parent, if applicable."""
|
47
|
-
self.cases[case.id] = CaseNode(value=case, parent_id=parent_id)
|
46
|
+
self.cases[case.id] = CaseNode(value=case, parent_id=parent_id, transition=transition)
|
48
47
|
|
49
48
|
def record_response(self, *, case_id: str, response: Response) -> None:
|
50
49
|
"""Record the API response for a given test case."""
|
@@ -94,30 +93,34 @@ class ScenarioRecorder:
|
|
94
93
|
return None
|
95
94
|
|
96
95
|
def find_related(self, *, case_id: str) -> Iterator[Case]:
|
97
|
-
"""Iterate over all
|
98
|
-
|
99
|
-
seen = {current_id}
|
96
|
+
"""Iterate over all cases in the tree, starting from the root."""
|
97
|
+
seen = {case_id}
|
100
98
|
|
99
|
+
# First, find the root by going up
|
100
|
+
current_id = case_id
|
101
101
|
while True:
|
102
102
|
current_node = self.cases.get(current_id)
|
103
103
|
if current_node is None or current_node.parent_id is None:
|
104
|
+
root_id = current_id
|
104
105
|
break
|
106
|
+
current_id = current_node.parent_id
|
105
107
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
if parent_id ==
|
108
|
+
# Then traverse the whole tree from root
|
109
|
+
def traverse(node_id: str) -> Iterator[Case]:
|
110
|
+
# Get all children
|
111
|
+
for case_id, node in self.cases.items():
|
112
|
+
if node.parent_id == node_id and case_id not in seen:
|
111
113
|
seen.add(case_id)
|
112
|
-
yield
|
114
|
+
yield node.value
|
115
|
+
# Recurse into children
|
116
|
+
yield from traverse(case_id)
|
113
117
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
yield parent_node.value
|
118
|
+
# Start traversal from root
|
119
|
+
root_node = self.cases.get(root_id)
|
120
|
+
if root_node and root_id not in seen:
|
121
|
+
seen.add(root_id)
|
122
|
+
yield root_node.value
|
123
|
+
yield from traverse(root_id)
|
121
124
|
|
122
125
|
def find_response(self, *, case_id: str) -> Response | None:
|
123
126
|
"""Retrieve the API response for a given test case, if available."""
|
@@ -133,8 +136,11 @@ class CaseNode:
|
|
133
136
|
|
134
137
|
value: Case
|
135
138
|
parent_id: str | None
|
139
|
+
# Transition may be absent if `parent_id` is present for cases when a case is derived inside a check
|
140
|
+
# and outside of the implemented transition logic (e.g. Open API links)
|
141
|
+
transition: Transition | None
|
136
142
|
|
137
|
-
__slots__ = ("value", "parent_id")
|
143
|
+
__slots__ = ("value", "parent_id", "transition")
|
138
144
|
|
139
145
|
|
140
146
|
@dataclass
|
schemathesis/errors.py
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
"""Public Schemathesis errors."""
|
2
2
|
|
3
|
-
from schemathesis.core.errors import
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
3
|
+
from schemathesis.core.errors import (
|
4
|
+
IncorrectUsage,
|
5
|
+
InternalError,
|
6
|
+
InvalidHeadersExample,
|
7
|
+
InvalidLinkDefinition,
|
8
|
+
InvalidRateLimit,
|
9
|
+
InvalidRegexPattern,
|
10
|
+
InvalidRegexType,
|
11
|
+
InvalidSchema,
|
12
|
+
LoaderError,
|
13
|
+
NoLinksFound,
|
14
|
+
OperationNotFound,
|
15
|
+
SchemathesisError,
|
16
|
+
SerializationError,
|
17
|
+
SerializationNotPossible,
|
18
|
+
UnboundPrefix,
|
19
|
+
)
|
16
20
|
|
17
21
|
__all__ = [
|
18
22
|
"IncorrectUsage",
|
19
23
|
"InternalError",
|
20
24
|
"InvalidHeadersExample",
|
25
|
+
"InvalidLinkDefinition",
|
21
26
|
"InvalidRateLimit",
|
22
27
|
"InvalidRegexPattern",
|
23
28
|
"InvalidRegexType",
|
24
29
|
"InvalidSchema",
|
25
30
|
"LoaderError",
|
26
31
|
"OperationNotFound",
|
32
|
+
"NoLinksFound",
|
27
33
|
"SchemathesisError",
|
28
34
|
"SerializationError",
|
29
35
|
"SerializationNotPossible",
|
@@ -194,6 +194,10 @@ class CoverageContext:
|
|
194
194
|
re.compile(pattern)
|
195
195
|
except re.error:
|
196
196
|
raise Unsatisfiable from None
|
197
|
+
if "minLength" in schema or "maxLength" in schema:
|
198
|
+
min_length = schema.get("minLength")
|
199
|
+
max_length = schema.get("maxLength")
|
200
|
+
pattern = update_quantifier(pattern, min_length, max_length)
|
197
201
|
return cached_draw(st.from_regex(pattern))
|
198
202
|
if (keys == ["items", "type"] or keys == ["items", "minItems", "type"]) and isinstance(schema["items"], dict):
|
199
203
|
items = schema["items"]
|
@@ -514,11 +518,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
514
518
|
# Default positive value
|
515
519
|
yield PositiveValue(ctx.generate_from_schema(schema), description="Valid string")
|
516
520
|
elif "pattern" in schema:
|
517
|
-
# Without merging `maxLength` & `minLength` into a regex it is problematic
|
518
|
-
# to generate a valid value as the unredlying machinery will resort to filtering
|
519
|
-
# and it is unlikely that it will generate a string of that length
|
520
521
|
yield PositiveValue(ctx.generate_from_schema(schema), description="Valid string")
|
521
|
-
return
|
522
522
|
|
523
523
|
seen = set()
|
524
524
|
|
@@ -386,19 +386,22 @@ def _iter_coverage_cases(
|
|
386
386
|
container = template["query"]
|
387
387
|
for parameter in operation.query:
|
388
388
|
instant = Instant()
|
389
|
-
value
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
389
|
+
# Could be absent if value schema can't be negated
|
390
|
+
# I.e. contains just `default` value without any other keywords
|
391
|
+
value = container.get(parameter.name, NOT_SET)
|
392
|
+
if value is not NOT_SET:
|
393
|
+
yield operation.Case(
|
394
|
+
**{**template, "query": {**container, parameter.name: [value, value]}},
|
395
|
+
meta=CaseMetadata(
|
396
|
+
generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
|
397
|
+
components={},
|
398
|
+
phase=PhaseInfo.coverage(
|
399
|
+
description=f"Duplicate `{parameter.name}` query parameter",
|
400
|
+
parameter=parameter.name,
|
401
|
+
parameter_location="query",
|
402
|
+
),
|
399
403
|
),
|
400
|
-
)
|
401
|
-
)
|
404
|
+
)
|
402
405
|
# Generate missing required parameters
|
403
406
|
for parameter in operation.iter_parameters():
|
404
407
|
if parameter.is_required and parameter.location != "path":
|
@@ -10,7 +10,8 @@ from hypothesis.errors import InvalidDefinition
|
|
10
10
|
from hypothesis.stateful import RuleBasedStateMachine
|
11
11
|
|
12
12
|
from schemathesis.checks import CheckFunction
|
13
|
-
from schemathesis.core.errors import
|
13
|
+
from schemathesis.core.errors import NoLinksFound
|
14
|
+
from schemathesis.core.result import Result
|
14
15
|
from schemathesis.core.transport import Response
|
15
16
|
from schemathesis.generation.case import Case
|
16
17
|
|
@@ -18,30 +19,64 @@ if TYPE_CHECKING:
|
|
18
19
|
import hypothesis
|
19
20
|
from requests.structures import CaseInsensitiveDict
|
20
21
|
|
21
|
-
from schemathesis.schemas import
|
22
|
+
from schemathesis.schemas import BaseSchema
|
22
23
|
|
23
24
|
|
24
|
-
|
25
|
-
"Stateful testing requires at least one OpenAPI link in the schema, but no links detected. "
|
26
|
-
"Please add OpenAPI links to enable stateful testing or use stateless tests instead. \n"
|
27
|
-
"See https://schemathesis.readthedocs.io/en/stable/stateful.html#how-to-specify-connections for more information."
|
28
|
-
)
|
29
|
-
|
25
|
+
DEFAULT_STATEFUL_STEP_COUNT = 6
|
30
26
|
DEFAULT_STATE_MACHINE_SETTINGS = hypothesis.settings(
|
31
27
|
phases=[hypothesis.Phase.generate],
|
32
28
|
deadline=None,
|
33
|
-
stateful_step_count=
|
29
|
+
stateful_step_count=DEFAULT_STATEFUL_STEP_COUNT,
|
34
30
|
suppress_health_check=list(hypothesis.HealthCheck),
|
35
31
|
)
|
36
32
|
|
37
33
|
|
38
34
|
@dataclass
|
39
|
-
class
|
35
|
+
class StepInput:
|
36
|
+
"""Input for a single state machine step."""
|
37
|
+
|
38
|
+
case: Case
|
39
|
+
transition: Transition | None # None for initial steps
|
40
|
+
|
41
|
+
__slots__ = ("case", "transition")
|
42
|
+
|
43
|
+
@classmethod
|
44
|
+
def initial(cls, case: Case) -> StepInput:
|
45
|
+
return cls(case=case, transition=None)
|
46
|
+
|
47
|
+
|
48
|
+
@dataclass
|
49
|
+
class Transition:
|
50
|
+
"""Data about transition execution."""
|
51
|
+
|
52
|
+
# ID of the transition (e.g. link name)
|
53
|
+
id: str
|
54
|
+
parent_id: str
|
55
|
+
parameters: dict[str, dict[str, ExtractedParam]]
|
56
|
+
request_body: ExtractedParam | None
|
57
|
+
|
58
|
+
__slots__ = ("id", "parent_id", "parameters", "request_body")
|
59
|
+
|
60
|
+
|
61
|
+
@dataclass
|
62
|
+
class ExtractedParam:
|
63
|
+
"""Result of parameter extraction."""
|
64
|
+
|
65
|
+
definition: Any
|
66
|
+
value: Result[Any, Exception]
|
67
|
+
|
68
|
+
__slots__ = ("definition", "value")
|
69
|
+
|
70
|
+
|
71
|
+
@dataclass
|
72
|
+
class StepOutput:
|
40
73
|
"""Output from a single transition of a state machine."""
|
41
74
|
|
42
75
|
response: Response
|
43
76
|
case: Case
|
44
77
|
|
78
|
+
__slots__ = ("response", "case")
|
79
|
+
|
45
80
|
|
46
81
|
def _normalize_name(name: str) -> str:
|
47
82
|
return re.sub(r"\W|^(?=\d)", "_", name).replace("__", "_")
|
@@ -64,7 +99,11 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
64
99
|
super().__init__() # type: ignore
|
65
100
|
except InvalidDefinition as exc:
|
66
101
|
if "defines no rules" in str(exc):
|
67
|
-
|
102
|
+
if not self.schema.statistic.links.total:
|
103
|
+
message = "Schema contains no link definitions required for stateful testing"
|
104
|
+
else:
|
105
|
+
message = "All link definitions required for stateful testing are excluded by filters"
|
106
|
+
raise NoLinksFound(message) from None
|
68
107
|
raise
|
69
108
|
self.setup()
|
70
109
|
|
@@ -89,10 +128,10 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
89
128
|
target = _normalize_name(target)
|
90
129
|
return super()._new_name(target) # type: ignore
|
91
130
|
|
92
|
-
def _get_target_for_result(self, result:
|
131
|
+
def _get_target_for_result(self, result: StepOutput) -> str | None:
|
93
132
|
raise NotImplementedError
|
94
133
|
|
95
|
-
def _add_result_to_targets(self, targets: tuple[str, ...], result:
|
134
|
+
def _add_result_to_targets(self, targets: tuple[str, ...], result: StepOutput | None) -> None:
|
96
135
|
if result is None:
|
97
136
|
return
|
98
137
|
target = self._get_target_for_result(result)
|
@@ -115,19 +154,11 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
115
154
|
# To provide the return type in the rendered documentation
|
116
155
|
teardown.__doc__ = RuleBasedStateMachine.teardown.__doc__
|
117
156
|
|
118
|
-
def
|
119
|
-
raise NotImplementedError
|
120
|
-
|
121
|
-
def _step(self, case: Case, previous: StepResult | None = None, link: Direction | None = None) -> StepResult | None:
|
122
|
-
# This method is a proxy that is used under the hood during the state machine initialization.
|
123
|
-
# The whole point of having it is to make it possible to override `step`; otherwise, custom "step" is ignored.
|
124
|
-
# It happens because, at the point of initialization, the final class is not yet created.
|
157
|
+
def _step(self, input: StepInput) -> StepOutput | None:
|
125
158
|
__tracebackhide__ = True
|
126
|
-
|
127
|
-
return self.step(case, (previous, link))
|
128
|
-
return self.step(case, None)
|
159
|
+
return self.step(input)
|
129
160
|
|
130
|
-
def step(self,
|
161
|
+
def step(self, input: StepInput) -> StepOutput:
|
131
162
|
"""A single state machine step.
|
132
163
|
|
133
164
|
:param Case case: Generated test case data that should be sent in an API call to the tested API operation.
|
@@ -137,15 +168,12 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
137
168
|
It is the most high-level point to extend the testing process. You probably don't need it in most cases.
|
138
169
|
"""
|
139
170
|
__tracebackhide__ = True
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
self.
|
144
|
-
|
145
|
-
response
|
146
|
-
self.after_call(response, case)
|
147
|
-
self.validate_response(response, case)
|
148
|
-
return self.store_result(response, case)
|
171
|
+
self.before_call(input.case)
|
172
|
+
kwargs = self.get_call_kwargs(input.case)
|
173
|
+
response = self.call(input.case, **kwargs)
|
174
|
+
self.after_call(response, input.case)
|
175
|
+
self.validate_response(response, input.case)
|
176
|
+
return StepOutput(response, input.case)
|
149
177
|
|
150
178
|
def before_call(self, case: Case) -> None:
|
151
179
|
"""Hook method for modifying the case data before making a request.
|
@@ -271,15 +299,3 @@ class APIStateMachine(RuleBasedStateMachine):
|
|
271
299
|
"""
|
272
300
|
__tracebackhide__ = True
|
273
301
|
case.validate_response(response, additional_checks=additional_checks)
|
274
|
-
|
275
|
-
def store_result(self, response: Response, case: Case) -> StepResult:
|
276
|
-
return StepResult(response, case)
|
277
|
-
|
278
|
-
|
279
|
-
class Direction:
|
280
|
-
name: str
|
281
|
-
status_code: str
|
282
|
-
operation: APIOperation
|
283
|
-
|
284
|
-
def set_data(self, case: Case, **kwargs: Any) -> None:
|
285
|
-
raise NotImplementedError
|
schemathesis/graphql/checks.py
CHANGED
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
|
|
11
11
|
class UnexpectedGraphQLResponse(Failure):
|
12
12
|
"""GraphQL response is not a JSON object."""
|
13
13
|
|
14
|
-
__slots__ = ("operation", "type_name", "title", "message", "
|
14
|
+
__slots__ = ("operation", "type_name", "title", "message", "case_id", "severity")
|
15
15
|
|
16
16
|
def __init__(
|
17
17
|
self,
|
@@ -20,14 +20,12 @@ class UnexpectedGraphQLResponse(Failure):
|
|
20
20
|
type_name: str,
|
21
21
|
title: str = "Unexpected GraphQL Response",
|
22
22
|
message: str,
|
23
|
-
code: str = "graphql_unexpected_response",
|
24
23
|
case_id: str | None = None,
|
25
24
|
) -> None:
|
26
25
|
self.operation = operation
|
27
26
|
self.type_name = type_name
|
28
27
|
self.title = title
|
29
28
|
self.message = message
|
30
|
-
self.code = code
|
31
29
|
self.case_id = case_id
|
32
30
|
self.severity = Severity.MEDIUM
|
33
31
|
|
@@ -39,7 +37,7 @@ class UnexpectedGraphQLResponse(Failure):
|
|
39
37
|
class GraphQLClientError(Failure):
|
40
38
|
"""GraphQL query has not been executed."""
|
41
39
|
|
42
|
-
__slots__ = ("operation", "errors", "title", "message", "
|
40
|
+
__slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
|
43
41
|
|
44
42
|
def __init__(
|
45
43
|
self,
|
@@ -48,14 +46,12 @@ class GraphQLClientError(Failure):
|
|
48
46
|
message: str,
|
49
47
|
errors: list[GraphQLFormattedError],
|
50
48
|
title: str = "GraphQL client error",
|
51
|
-
code: str = "graphql_client_error",
|
52
49
|
case_id: str | None = None,
|
53
50
|
) -> None:
|
54
51
|
self.operation = operation
|
55
52
|
self.errors = errors
|
56
53
|
self.title = title
|
57
54
|
self.message = message
|
58
|
-
self.code = code
|
59
55
|
self.case_id = case_id
|
60
56
|
self._unique_key_cache: str | None = None
|
61
57
|
self.severity = Severity.MEDIUM
|
@@ -70,7 +66,7 @@ class GraphQLClientError(Failure):
|
|
70
66
|
class GraphQLServerError(Failure):
|
71
67
|
"""GraphQL response indicates at least one server error."""
|
72
68
|
|
73
|
-
__slots__ = ("operation", "errors", "title", "message", "
|
69
|
+
__slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
|
74
70
|
|
75
71
|
def __init__(
|
76
72
|
self,
|
@@ -79,14 +75,12 @@ class GraphQLServerError(Failure):
|
|
79
75
|
message: str,
|
80
76
|
errors: list[GraphQLFormattedError],
|
81
77
|
title: str = "GraphQL server error",
|
82
|
-
code: str = "graphql_server_error",
|
83
78
|
case_id: str | None = None,
|
84
79
|
) -> None:
|
85
80
|
self.operation = operation
|
86
81
|
self.errors = errors
|
87
82
|
self.title = title
|
88
83
|
self.message = message
|
89
|
-
self.code = code
|
90
84
|
self.case_id = case_id
|
91
85
|
self._unique_key_cache: str | None = None
|
92
86
|
self.severity = Severity.CRITICAL
|