schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/cli/__init__.py +3 -3
- schemathesis/cli/commands/run/__init__.py +148 -94
- schemathesis/cli/commands/run/context.py +72 -2
- schemathesis/cli/commands/run/executor.py +32 -12
- schemathesis/cli/commands/run/filters.py +1 -0
- schemathesis/cli/commands/run/handlers/cassettes.py +27 -46
- schemathesis/cli/commands/run/handlers/junitxml.py +1 -1
- schemathesis/cli/commands/run/handlers/output.py +72 -16
- schemathesis/cli/commands/run/hypothesis.py +30 -19
- schemathesis/cli/commands/run/reports.py +72 -0
- schemathesis/cli/commands/run/validation.py +18 -12
- schemathesis/cli/ext/groups.py +42 -13
- schemathesis/cli/ext/options.py +15 -8
- schemathesis/core/errors.py +79 -11
- schemathesis/core/failures.py +2 -1
- schemathesis/core/transforms.py +1 -1
- schemathesis/engine/errors.py +8 -3
- schemathesis/engine/phases/stateful/_executor.py +1 -1
- schemathesis/engine/phases/unit/__init__.py +2 -3
- schemathesis/engine/phases/unit/_executor.py +16 -13
- schemathesis/errors.py +6 -2
- schemathesis/filters.py +8 -0
- schemathesis/generation/coverage.py +6 -1
- schemathesis/generation/stateful/state_machine.py +49 -3
- schemathesis/pytest/lazy.py +2 -3
- schemathesis/pytest/plugin.py +2 -3
- schemathesis/schemas.py +1 -1
- schemathesis/specs/openapi/checks.py +27 -10
- schemathesis/specs/openapi/expressions/__init__.py +22 -6
- schemathesis/specs/openapi/expressions/nodes.py +15 -21
- schemathesis/specs/openapi/expressions/parser.py +1 -1
- schemathesis/specs/openapi/parameters.py +0 -2
- schemathesis/specs/openapi/schemas.py +13 -13
- schemathesis/specs/openapi/stateful/__init__.py +96 -23
- schemathesis/specs/openapi/{links.py → stateful/links.py} +60 -16
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/METADATA +1 -1
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/RECORD +40 -39
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a3.dist-info → schemathesis-4.0.0a4.dist-info}/licenses/LICENSE +0 -0
schemathesis/pytest/plugin.py
CHANGED
@@ -149,11 +149,10 @@ class SchemathesisCase(PyCollector):
|
|
149
149
|
error = result.err()
|
150
150
|
funcobj = error.as_failing_test_function()
|
151
151
|
name = self.name
|
152
|
-
# `full_path` is always available in this case
|
153
152
|
if error.method:
|
154
|
-
name += f"[{error.method.upper()} {error.
|
153
|
+
name += f"[{error.method.upper()} {error.path}]"
|
155
154
|
else:
|
156
|
-
name += f"[{error.
|
155
|
+
name += f"[{error.path}]"
|
157
156
|
|
158
157
|
cls = self._get_class_parent()
|
159
158
|
definition: FunctionDefinition = FunctionDefinition.from_parent(
|
schemathesis/schemas.py
CHANGED
@@ -619,7 +619,7 @@ class APIOperation(Generic[P]):
|
|
619
619
|
|
620
620
|
def __post_init__(self) -> None:
|
621
621
|
if self.label is None:
|
622
|
-
self.label = f"{self.method.upper()} {self.
|
622
|
+
self.label = f"{self.method.upper()} {self.path}" # type: ignore
|
623
623
|
|
624
624
|
@property
|
625
625
|
def full_path(self) -> str:
|
@@ -41,11 +41,20 @@ if TYPE_CHECKING:
|
|
41
41
|
from ...schemas import APIOperation
|
42
42
|
|
43
43
|
|
44
|
+
def is_unexpected_http_status_case(case: Case) -> bool:
|
45
|
+
# Skip checks for requests using HTTP methods not defined in the API spec
|
46
|
+
return bool(
|
47
|
+
case.meta
|
48
|
+
and isinstance(case.meta.phase.data, CoveragePhaseData)
|
49
|
+
and case.meta.phase.data.description.startswith("Unspecified HTTP method")
|
50
|
+
)
|
51
|
+
|
52
|
+
|
44
53
|
@schemathesis.check
|
45
54
|
def status_code_conformance(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
46
55
|
from .schemas import BaseOpenAPISchema
|
47
56
|
|
48
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
57
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
49
58
|
return True
|
50
59
|
responses = case.operation.definition.raw.get("responses", {})
|
51
60
|
# "default" can be used as the default response object for all HTTP codes that are not covered individually
|
@@ -74,7 +83,7 @@ def _expand_responses(responses: dict[str | int, Any]) -> Generator[int, None, N
|
|
74
83
|
def content_type_conformance(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
75
84
|
from .schemas import BaseOpenAPISchema
|
76
85
|
|
77
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
86
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
78
87
|
return True
|
79
88
|
documented_content_types = case.operation.schema.get_content_types(case.operation, response)
|
80
89
|
if not documented_content_types:
|
@@ -128,7 +137,7 @@ def response_headers_conformance(ctx: CheckContext, response: Response, case: Ca
|
|
128
137
|
from .parameters import OpenAPI20Parameter, OpenAPI30Parameter
|
129
138
|
from .schemas import BaseOpenAPISchema, OpenApi30, _maybe_raise_one_or_more
|
130
139
|
|
131
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
140
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
132
141
|
return True
|
133
142
|
resolved = case.operation.schema.get_headers(case.operation, response)
|
134
143
|
if not resolved:
|
@@ -208,7 +217,7 @@ def _coerce_header_value(value: str, schema: dict[str, Any]) -> str | int | floa
|
|
208
217
|
def response_schema_conformance(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
209
218
|
from .schemas import BaseOpenAPISchema
|
210
219
|
|
211
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
220
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
212
221
|
return True
|
213
222
|
return case.operation.validate_response(response)
|
214
223
|
|
@@ -217,7 +226,11 @@ def response_schema_conformance(ctx: CheckContext, response: Response, case: Cas
|
|
217
226
|
def negative_data_rejection(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
218
227
|
from .schemas import BaseOpenAPISchema
|
219
228
|
|
220
|
-
if
|
229
|
+
if (
|
230
|
+
not isinstance(case.operation.schema, BaseOpenAPISchema)
|
231
|
+
or case.meta is None
|
232
|
+
or is_unexpected_http_status_case(case)
|
233
|
+
):
|
221
234
|
return True
|
222
235
|
|
223
236
|
config = ctx.config.get(negative_data_rejection, NegativeDataRejectionConfig())
|
@@ -241,7 +254,11 @@ def negative_data_rejection(ctx: CheckContext, response: Response, case: Case) -
|
|
241
254
|
def positive_data_acceptance(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
242
255
|
from .schemas import BaseOpenAPISchema
|
243
256
|
|
244
|
-
if
|
257
|
+
if (
|
258
|
+
not isinstance(case.operation.schema, BaseOpenAPISchema)
|
259
|
+
or case.meta is None
|
260
|
+
or is_unexpected_http_status_case(case)
|
261
|
+
):
|
245
262
|
return True
|
246
263
|
|
247
264
|
config = ctx.config.get(positive_data_acceptance, PositiveDataAcceptanceConfig())
|
@@ -260,7 +277,7 @@ def positive_data_acceptance(ctx: CheckContext, response: Response, case: Case)
|
|
260
277
|
def missing_required_header(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
261
278
|
# NOTE: This check is intentionally not registered with `@schemathesis.check` because it is experimental
|
262
279
|
meta = case.meta
|
263
|
-
if meta is None or not isinstance(meta.phase.data, CoveragePhaseData):
|
280
|
+
if meta is None or not isinstance(meta.phase.data, CoveragePhaseData) or is_unexpected_http_status_case(case):
|
264
281
|
return None
|
265
282
|
data = meta.phase.data
|
266
283
|
if (
|
@@ -332,7 +349,7 @@ def has_only_additional_properties_in_non_body_parameters(case: Case) -> bool:
|
|
332
349
|
def use_after_free(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
333
350
|
from .schemas import BaseOpenAPISchema
|
334
351
|
|
335
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
352
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
336
353
|
return True
|
337
354
|
if response.status_code == 404 or response.status_code >= 500:
|
338
355
|
return None
|
@@ -373,7 +390,7 @@ def use_after_free(ctx: CheckContext, response: Response, case: Case) -> bool |
|
|
373
390
|
def ensure_resource_availability(ctx: CheckContext, response: Response, case: Case) -> bool | None:
|
374
391
|
from .schemas import BaseOpenAPISchema
|
375
392
|
|
376
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
393
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
377
394
|
return True
|
378
395
|
|
379
396
|
# First, check if this is a 4XX response
|
@@ -453,7 +470,7 @@ def ignored_auth(ctx: CheckContext, response: Response, case: Case) -> bool | No
|
|
453
470
|
"""Check if an operation declares authentication as a requirement but does not actually enforce it."""
|
454
471
|
from .schemas import BaseOpenAPISchema
|
455
472
|
|
456
|
-
if not isinstance(case.operation.schema, BaseOpenAPISchema):
|
473
|
+
if not isinstance(case.operation.schema, BaseOpenAPISchema) or is_unexpected_http_status_case(case):
|
457
474
|
return True
|
458
475
|
security_parameters = _get_security_parameters(case.operation)
|
459
476
|
# Authentication is required for this API operation and response is successful
|
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|
8
8
|
import json
|
9
9
|
from typing import Any
|
10
10
|
|
11
|
+
from schemathesis.core.transforms import UNRESOLVABLE, Unresolvable
|
11
12
|
from schemathesis.generation.stateful.state_machine import StepOutput
|
12
13
|
|
13
14
|
from . import lexer, nodes, parser
|
@@ -25,21 +26,36 @@ def evaluate(expr: Any, output: StepOutput, evaluate_nested: bool = False) -> An
|
|
25
26
|
parts = [node.evaluate(output) for node in parser.parse(expr)]
|
26
27
|
if len(parts) == 1:
|
27
28
|
return parts[0] # keep the return type the same as the internal value type
|
28
|
-
|
29
|
+
if any(isinstance(part, Unresolvable) for part in parts):
|
30
|
+
return UNRESOLVABLE
|
29
31
|
return "".join(str(part) for part in parts if part is not None)
|
30
32
|
|
31
33
|
|
32
34
|
def _evaluate_nested(expr: dict[str, Any] | list, output: StepOutput) -> Any:
|
33
35
|
if isinstance(expr, dict):
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
result_dict = {}
|
37
|
+
for key, value in expr.items():
|
38
|
+
new_key = _evaluate_object_key(key, output)
|
39
|
+
if new_key is UNRESOLVABLE:
|
40
|
+
return new_key
|
41
|
+
new_value = evaluate(value, output, evaluate_nested=True)
|
42
|
+
if new_value is UNRESOLVABLE:
|
43
|
+
return new_value
|
44
|
+
result_dict[new_key] = new_value
|
45
|
+
return result_dict
|
46
|
+
result_list = []
|
47
|
+
for item in expr:
|
48
|
+
new_value = evaluate(item, output, evaluate_nested=True)
|
49
|
+
if new_value is UNRESOLVABLE:
|
50
|
+
return new_value
|
51
|
+
result_list.append(new_value)
|
52
|
+
return result_list
|
39
53
|
|
40
54
|
|
41
55
|
def _evaluate_object_key(key: str, output: StepOutput) -> Any:
|
42
56
|
evaluated = evaluate(key, output)
|
57
|
+
if evaluated is UNRESOLVABLE:
|
58
|
+
return evaluated
|
43
59
|
if isinstance(evaluated, str):
|
44
60
|
return evaluated
|
45
61
|
if isinstance(evaluated, bool):
|
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, cast
|
|
8
8
|
|
9
9
|
from requests.structures import CaseInsensitiveDict
|
10
10
|
|
11
|
-
from schemathesis.core.transforms import UNRESOLVABLE, resolve_pointer
|
11
|
+
from schemathesis.core.transforms import UNRESOLVABLE, Unresolvable, resolve_pointer
|
12
12
|
from schemathesis.generation.stateful.state_machine import StepOutput
|
13
13
|
from schemathesis.transport.requests import REQUESTS_TRANSPORT
|
14
14
|
|
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
|
20
20
|
class Node:
|
21
21
|
"""Generic expression node."""
|
22
22
|
|
23
|
-
def evaluate(self, output: StepOutput) -> str:
|
23
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
24
24
|
raise NotImplementedError
|
25
25
|
|
26
26
|
|
@@ -39,7 +39,7 @@ class String(Node):
|
|
39
39
|
|
40
40
|
value: str
|
41
41
|
|
42
|
-
def evaluate(self, output: StepOutput) -> str:
|
42
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
43
43
|
"""String tokens are passed as they are.
|
44
44
|
|
45
45
|
``foo{$request.path.id}``
|
@@ -53,7 +53,7 @@ class String(Node):
|
|
53
53
|
class URL(Node):
|
54
54
|
"""A node for `$url` expression."""
|
55
55
|
|
56
|
-
def evaluate(self, output: StepOutput) -> str:
|
56
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
57
57
|
import requests
|
58
58
|
|
59
59
|
base_url = output.case.operation.base_url or "http://127.0.0.1"
|
@@ -66,7 +66,7 @@ class URL(Node):
|
|
66
66
|
class Method(Node):
|
67
67
|
"""A node for `$method` expression."""
|
68
68
|
|
69
|
-
def evaluate(self, output: StepOutput) -> str:
|
69
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
70
70
|
return output.case.operation.method.upper()
|
71
71
|
|
72
72
|
|
@@ -74,7 +74,7 @@ class Method(Node):
|
|
74
74
|
class StatusCode(Node):
|
75
75
|
"""A node for `$statusCode` expression."""
|
76
76
|
|
77
|
-
def evaluate(self, output: StepOutput) -> str:
|
77
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
78
78
|
return str(output.response.status_code)
|
79
79
|
|
80
80
|
|
@@ -86,7 +86,7 @@ class NonBodyRequest(Node):
|
|
86
86
|
parameter: str
|
87
87
|
extractor: Extractor | None = None
|
88
88
|
|
89
|
-
def evaluate(self, output: StepOutput) -> str:
|
89
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
90
90
|
container: dict | CaseInsensitiveDict = {
|
91
91
|
"query": output.case.query,
|
92
92
|
"path": output.case.path_parameters,
|
@@ -96,9 +96,9 @@ class NonBodyRequest(Node):
|
|
96
96
|
container = CaseInsensitiveDict(container)
|
97
97
|
value = container.get(self.parameter)
|
98
98
|
if value is None:
|
99
|
-
return
|
99
|
+
return UNRESOLVABLE
|
100
100
|
if self.extractor is not None:
|
101
|
-
return self.extractor.extract(value) or
|
101
|
+
return self.extractor.extract(value) or UNRESOLVABLE
|
102
102
|
return value
|
103
103
|
|
104
104
|
|
@@ -108,14 +108,11 @@ class BodyRequest(Node):
|
|
108
108
|
|
109
109
|
pointer: str | None = None
|
110
110
|
|
111
|
-
def evaluate(self, output: StepOutput) -> Any:
|
111
|
+
def evaluate(self, output: StepOutput) -> Any | Unresolvable:
|
112
112
|
document = output.case.body
|
113
113
|
if self.pointer is None:
|
114
114
|
return document
|
115
|
-
|
116
|
-
if resolved is UNRESOLVABLE:
|
117
|
-
return None
|
118
|
-
return resolved
|
115
|
+
return resolve_pointer(document, self.pointer[1:])
|
119
116
|
|
120
117
|
|
121
118
|
@dataclass
|
@@ -125,12 +122,12 @@ class HeaderResponse(Node):
|
|
125
122
|
parameter: str
|
126
123
|
extractor: Extractor | None = None
|
127
124
|
|
128
|
-
def evaluate(self, output: StepOutput) -> str:
|
125
|
+
def evaluate(self, output: StepOutput) -> str | Unresolvable:
|
129
126
|
value = output.response.headers.get(self.parameter.lower())
|
130
127
|
if value is None:
|
131
|
-
return
|
128
|
+
return UNRESOLVABLE
|
132
129
|
if self.extractor is not None:
|
133
|
-
return self.extractor.extract(value[0]) or
|
130
|
+
return self.extractor.extract(value[0]) or UNRESOLVABLE
|
134
131
|
return value[0]
|
135
132
|
|
136
133
|
|
@@ -145,7 +142,4 @@ class BodyResponse(Node):
|
|
145
142
|
if self.pointer is None:
|
146
143
|
# We need the parsed document - data will be serialized before sending to the application
|
147
144
|
return document
|
148
|
-
|
149
|
-
if resolved is UNRESOLVABLE:
|
150
|
-
return None
|
151
|
-
return resolved
|
145
|
+
return resolve_pointer(document, self.pointer[1:])
|
@@ -46,7 +46,7 @@ def _parse_variable(tokens: lexer.TokenGenerator, token: lexer.Token, expr: str)
|
|
46
46
|
elif token.value == nodes.NodeType.RESPONSE.value:
|
47
47
|
yield _parse_response(tokens, expr)
|
48
48
|
else:
|
49
|
-
raise UnknownToken(token.value)
|
49
|
+
raise UnknownToken(f"Invalid expression `{expr}`. Unknown token: `{token.value}`")
|
50
50
|
|
51
51
|
|
52
52
|
def _parse_request(tokens: lexer.TokenGenerator, expr: str) -> nodes.BodyRequest | nodes.NonBodyRequest:
|
@@ -376,7 +376,6 @@ def get_parameter_schema(operation: APIOperation, data: dict[str, Any]) -> dict[
|
|
376
376
|
),
|
377
377
|
path=operation.path,
|
378
378
|
method=operation.method,
|
379
|
-
full_path=operation.full_path,
|
380
379
|
)
|
381
380
|
return data["schema"]
|
382
381
|
# https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#fixed-fields-10
|
@@ -388,7 +387,6 @@ def get_parameter_schema(operation: APIOperation, data: dict[str, Any]) -> dict[
|
|
388
387
|
MISSING_SCHEMA_OR_CONTENT_MESSAGE.format(location=data.get("in", ""), name=data.get("name", "<UNKNOWN>")),
|
389
388
|
path=operation.path,
|
390
389
|
method=operation.method,
|
391
|
-
full_path=operation.full_path,
|
392
390
|
) from exc
|
393
391
|
options = iter(content.values())
|
394
392
|
media_type_object = next(options)
|
@@ -41,11 +41,12 @@ from schemathesis.generation.case import Case
|
|
41
41
|
from schemathesis.generation.meta import CaseMetadata
|
42
42
|
from schemathesis.generation.overrides import Override, OverrideMark, check_no_override_mark
|
43
43
|
from schemathesis.openapi.checks import JsonSchemaError, MissingContentType
|
44
|
+
from schemathesis.specs.openapi.stateful import links
|
44
45
|
|
45
46
|
from ...generation import GenerationConfig, GenerationMode
|
46
47
|
from ...hooks import HookContext, HookDispatcher
|
47
48
|
from ...schemas import APIOperation, APIOperationMap, ApiStatistic, BaseSchema, OperationDefinition
|
48
|
-
from . import
|
49
|
+
from . import serialization
|
49
50
|
from ._cache import OperationCache
|
50
51
|
from ._hypothesis import openapi_cases
|
51
52
|
from .converter import to_json_schema, to_json_schema_recursive
|
@@ -155,7 +156,6 @@ class BaseOpenAPISchema(BaseSchema):
|
|
155
156
|
return True
|
156
157
|
if self.filter_set.is_empty():
|
157
158
|
return False
|
158
|
-
path = self.get_full_path(path)
|
159
159
|
# Attribute assignment is way faster than creating a new namespace every time
|
160
160
|
operation = _ctx_cache.operation
|
161
161
|
operation.method = method
|
@@ -365,28 +365,24 @@ class BaseOpenAPISchema(BaseSchema):
|
|
365
365
|
def _into_err(self, error: Exception, path: str | None, method: str | None) -> Err[InvalidSchema]:
|
366
366
|
__tracebackhide__ = True
|
367
367
|
try:
|
368
|
-
|
369
|
-
self._raise_invalid_schema(error, full_path, path, method)
|
368
|
+
self._raise_invalid_schema(error, path, method)
|
370
369
|
except InvalidSchema as exc:
|
371
370
|
return Err(exc)
|
372
371
|
|
373
372
|
def _raise_invalid_schema(
|
374
373
|
self,
|
375
374
|
error: Exception,
|
376
|
-
full_path: str | None = None,
|
377
375
|
path: str | None = None,
|
378
376
|
method: str | None = None,
|
379
377
|
) -> NoReturn:
|
380
378
|
__tracebackhide__ = True
|
381
379
|
if isinstance(error, RefResolutionError):
|
382
|
-
raise InvalidSchema.from_reference_resolution_error(
|
383
|
-
error, path=path, method=method, full_path=full_path
|
384
|
-
) from None
|
380
|
+
raise InvalidSchema.from_reference_resolution_error(error, path=path, method=method) from None
|
385
381
|
try:
|
386
382
|
self.validate()
|
387
383
|
except jsonschema.ValidationError as exc:
|
388
|
-
raise InvalidSchema.from_jsonschema_error(exc, path=path, method=method
|
389
|
-
raise InvalidSchema(SCHEMA_ERROR_MESSAGE, path=path, method=method
|
384
|
+
raise InvalidSchema.from_jsonschema_error(exc, path=path, method=method) from None
|
385
|
+
raise InvalidSchema(SCHEMA_ERROR_MESSAGE, path=path, method=method) from error
|
390
386
|
|
391
387
|
def validate(self) -> None:
|
392
388
|
with suppress(TypeError):
|
@@ -584,8 +580,7 @@ class BaseOpenAPISchema(BaseSchema):
|
|
584
580
|
responses = operation.definition.raw["responses"]
|
585
581
|
except KeyError as exc:
|
586
582
|
path = operation.path
|
587
|
-
|
588
|
-
self._raise_invalid_schema(exc, full_path, path, operation.method)
|
583
|
+
self._raise_invalid_schema(exc, path, operation.method)
|
589
584
|
status_code = str(response.status_code)
|
590
585
|
if status_code in responses:
|
591
586
|
return self.resolver.resolve_in_scope(responses[status_code], operation.definition.scope)
|
@@ -654,7 +649,12 @@ class BaseOpenAPISchema(BaseSchema):
|
|
654
649
|
def get_links(self, operation: APIOperation) -> dict[str, dict[str, Any]]:
|
655
650
|
result: dict[str, dict[str, Any]] = defaultdict(dict)
|
656
651
|
for status_code, link in links.get_all_links(operation):
|
657
|
-
|
652
|
+
if isinstance(link, Ok):
|
653
|
+
name = link.ok().name
|
654
|
+
else:
|
655
|
+
name = link.err().name
|
656
|
+
result[status_code][name] = link
|
657
|
+
|
658
658
|
return result
|
659
659
|
|
660
660
|
def get_tags(self, operation: APIOperation) -> list[str] | None:
|
@@ -7,15 +7,17 @@ from typing import TYPE_CHECKING, Any, Callable, Iterator
|
|
7
7
|
from hypothesis import strategies as st
|
8
8
|
from hypothesis.stateful import Bundle, Rule, precondition, rule
|
9
9
|
|
10
|
+
from schemathesis.core.errors import InvalidStateMachine
|
10
11
|
from schemathesis.core.result import Ok
|
12
|
+
from schemathesis.core.transforms import UNRESOLVABLE
|
11
13
|
from schemathesis.engine.recorder import ScenarioRecorder
|
12
14
|
from schemathesis.generation import GenerationMode
|
13
15
|
from schemathesis.generation.case import Case
|
14
16
|
from schemathesis.generation.hypothesis import strategies
|
15
17
|
from schemathesis.generation.stateful.state_machine import APIStateMachine, StepInput, StepOutput, _normalize_name
|
16
18
|
from schemathesis.schemas import APIOperation
|
17
|
-
from schemathesis.specs.openapi.links import OpenApiLink, get_all_links
|
18
19
|
from schemathesis.specs.openapi.stateful.control import TransitionController
|
20
|
+
from schemathesis.specs.openapi.stateful.links import OpenApiLink, get_all_links
|
19
21
|
from schemathesis.specs.openapi.utils import expand_status_code
|
20
22
|
|
21
23
|
if TYPE_CHECKING:
|
@@ -72,15 +74,35 @@ class ApiTransitions:
|
|
72
74
|
self.operations.setdefault(link.target.label, OperationTransitions()).incoming.append(link)
|
73
75
|
|
74
76
|
|
77
|
+
@dataclass
|
78
|
+
class RootTransitions:
|
79
|
+
"""Classification of API operations that can serve as entry points."""
|
80
|
+
|
81
|
+
__slots__ = ("reliable", "fallback")
|
82
|
+
|
83
|
+
def __init__(self) -> None:
|
84
|
+
# Operations likely to succeed and provide data for other transitions
|
85
|
+
self.reliable: set[str] = set()
|
86
|
+
# Operations that might work but are less reliable
|
87
|
+
self.fallback: set[str] = set()
|
88
|
+
|
89
|
+
|
75
90
|
def collect_transitions(operations: list[APIOperation]) -> ApiTransitions:
|
76
91
|
"""Collect all transitions between operations."""
|
77
92
|
transitions = ApiTransitions()
|
78
93
|
|
79
94
|
selected_labels = {operation.label for operation in operations}
|
95
|
+
errors = []
|
80
96
|
for operation in operations:
|
81
97
|
for _, link in get_all_links(operation):
|
82
|
-
if link
|
83
|
-
|
98
|
+
if isinstance(link, Ok):
|
99
|
+
if link.ok().target.label in selected_labels:
|
100
|
+
transitions.add_outgoing(operation.label, link.ok())
|
101
|
+
else:
|
102
|
+
errors.append(link.err())
|
103
|
+
|
104
|
+
if errors:
|
105
|
+
raise InvalidStateMachine(errors)
|
84
106
|
|
85
107
|
return transitions
|
86
108
|
|
@@ -109,15 +131,37 @@ def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
|
|
109
131
|
rules = {}
|
110
132
|
catch_all = Bundle("catch_all")
|
111
133
|
|
134
|
+
# We want stateful testing to be effective and focus on meaningful transitions.
|
135
|
+
# An operation is considered as a "root" transition (entry point) if it satisfies certain criteria
|
136
|
+
# that indicate it's likely to succeed and provide data for other transitions.
|
137
|
+
# For example:
|
138
|
+
# - POST operations that create resources
|
139
|
+
# - GET operations without path parameters (e.g., GET /users/ to list all users)
|
140
|
+
#
|
141
|
+
# We avoid adding operations as roots if they:
|
142
|
+
# 1. Have incoming transitions that will provide proper data
|
143
|
+
# Example: If POST /users/ -> GET /users/{id} exists, we don't need
|
144
|
+
# to generate random user IDs for GET /users/{id}
|
145
|
+
# 2. Are unlikely to succeed with random data
|
146
|
+
# Example: GET /users/{id} with random ID is likely to return 404
|
147
|
+
#
|
148
|
+
# This way we:
|
149
|
+
# 1. Maximize the chance of successful transitions
|
150
|
+
# 2. Don't waste the test budget (limited number of steps) on likely-to-fail operations
|
151
|
+
# 3. Focus on transitions that are designed to work together via links
|
152
|
+
|
153
|
+
roots = classify_root_transitions(operations, transitions)
|
154
|
+
|
112
155
|
for target in operations:
|
113
156
|
if target.label in transitions.operations:
|
114
157
|
incoming = transitions.operations[target.label].incoming
|
115
158
|
if incoming:
|
116
159
|
for link in incoming:
|
117
160
|
bundle_name = f"{link.source.label} -> {link.status_code}"
|
118
|
-
name = _normalize_name(
|
119
|
-
|
120
|
-
|
161
|
+
name = _normalize_name(
|
162
|
+
f"{link.source.label} -> {link.status_code} -> {link.name} -> {target.label}"
|
163
|
+
)
|
164
|
+
assert name not in rules, name
|
121
165
|
rules[name] = precondition(is_transition_allowed(bundle_name, link.source.label, target.label))(
|
122
166
|
transition(
|
123
167
|
name=name,
|
@@ -127,13 +171,8 @@ def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
|
|
127
171
|
),
|
128
172
|
)
|
129
173
|
)
|
130
|
-
if
|
131
|
-
|
132
|
-
# This approach also includes cases when there is an incoming transition back to POST
|
133
|
-
# For example, POST /users/ -> GET /users/{id}/
|
134
|
-
# The source operation has no prerequisite, but we need to allow this rule to be executed
|
135
|
-
# in order to reach other transitions
|
136
|
-
name = _normalize_name(f"{target.label} -> X")
|
174
|
+
if target.label in roots.reliable or (not roots.reliable and target.label in roots.fallback):
|
175
|
+
name = _normalize_name(f"RANDOM -> {target.label}")
|
137
176
|
if len(schema.generation_config.modes) == 1:
|
138
177
|
case_strategy = target.as_strategy(generation_mode=schema.generation_config.modes[0])
|
139
178
|
else:
|
@@ -168,41 +207,75 @@ def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
|
|
168
207
|
)
|
169
208
|
|
170
209
|
|
210
|
+
def classify_root_transitions(operations: list[APIOperation], transitions: ApiTransitions) -> RootTransitions:
|
211
|
+
"""Find operations that can serve as root transitions."""
|
212
|
+
roots = RootTransitions()
|
213
|
+
|
214
|
+
for operation in operations:
|
215
|
+
# Skip if operation has no outgoing transitions
|
216
|
+
operation_transitions = transitions.operations.get(operation.label)
|
217
|
+
if not operation_transitions or not operation_transitions.outgoing:
|
218
|
+
continue
|
219
|
+
|
220
|
+
if is_likely_root_transition(operation, operation_transitions):
|
221
|
+
roots.reliable.add(operation.label)
|
222
|
+
else:
|
223
|
+
roots.fallback.add(operation.label)
|
224
|
+
|
225
|
+
return roots
|
226
|
+
|
227
|
+
|
228
|
+
def is_likely_root_transition(operation: APIOperation, transitions: OperationTransitions) -> bool:
|
229
|
+
"""Check if operation is likely to succeed as a root transition."""
|
230
|
+
# POST operations with request bodies are likely to create resources
|
231
|
+
if operation.method == "post" and operation.body:
|
232
|
+
return True
|
233
|
+
|
234
|
+
# GET operations without path parameters are likely to return lists
|
235
|
+
if operation.method == "get" and not operation.path_parameters:
|
236
|
+
return True
|
237
|
+
|
238
|
+
return False
|
239
|
+
|
240
|
+
|
171
241
|
def into_step_input(
|
172
242
|
target: APIOperation, link: OpenApiLink, modes: list[GenerationMode]
|
173
243
|
) -> Callable[[StepOutput], st.SearchStrategy[StepInput]]:
|
174
244
|
def builder(_output: StepOutput) -> st.SearchStrategy[StepInput]:
|
175
245
|
@st.composite # type: ignore[misc]
|
176
246
|
def inner(draw: st.DrawFn, output: StepOutput) -> StepInput:
|
177
|
-
|
247
|
+
transition = link.extract(output)
|
178
248
|
|
179
249
|
kwargs: dict[str, Any] = {
|
180
250
|
container: {
|
181
251
|
name: extracted.value.ok()
|
182
252
|
for name, extracted in data.items()
|
183
|
-
if isinstance(extracted.value, Ok) and extracted.value.ok()
|
253
|
+
if isinstance(extracted.value, Ok) and extracted.value.ok() not in (None, UNRESOLVABLE)
|
184
254
|
}
|
185
|
-
for container, data in
|
255
|
+
for container, data in transition.parameters.items()
|
186
256
|
}
|
257
|
+
|
187
258
|
if (
|
188
|
-
|
189
|
-
and isinstance(
|
259
|
+
transition.request_body is not None
|
260
|
+
and isinstance(transition.request_body.value, Ok)
|
261
|
+
and transition.request_body.value.ok() is not UNRESOLVABLE
|
190
262
|
and not link.merge_body
|
191
263
|
):
|
192
|
-
kwargs["body"] =
|
264
|
+
kwargs["body"] = transition.request_body.value.ok()
|
193
265
|
cases = strategies.combine([target.as_strategy(generation_mode=mode, **kwargs) for mode in modes])
|
194
266
|
case = draw(cases)
|
195
267
|
if (
|
196
|
-
|
197
|
-
and isinstance(
|
268
|
+
transition.request_body is not None
|
269
|
+
and isinstance(transition.request_body.value, Ok)
|
270
|
+
and transition.request_body.value.ok() is not UNRESOLVABLE
|
198
271
|
and link.merge_body
|
199
272
|
):
|
200
|
-
new =
|
273
|
+
new = transition.request_body.value.ok()
|
201
274
|
if isinstance(case.body, dict) and isinstance(new, dict):
|
202
275
|
case.body = {**case.body, **new}
|
203
276
|
else:
|
204
277
|
case.body = new
|
205
|
-
return StepInput(case=case, transition=
|
278
|
+
return StepInput(case=case, transition=transition)
|
206
279
|
|
207
280
|
return inner(output=_output)
|
208
281
|
|