schemathesis 4.0.0a1__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. schemathesis/checks.py +6 -4
  2. schemathesis/cli/__init__.py +12 -1
  3. schemathesis/cli/commands/run/__init__.py +4 -4
  4. schemathesis/cli/commands/run/events.py +19 -4
  5. schemathesis/cli/commands/run/executor.py +9 -3
  6. schemathesis/cli/commands/run/filters.py +27 -19
  7. schemathesis/cli/commands/run/handlers/base.py +1 -1
  8. schemathesis/cli/commands/run/handlers/cassettes.py +1 -3
  9. schemathesis/cli/commands/run/handlers/output.py +860 -201
  10. schemathesis/cli/commands/run/validation.py +1 -1
  11. schemathesis/cli/ext/options.py +4 -1
  12. schemathesis/core/errors.py +8 -0
  13. schemathesis/core/failures.py +54 -24
  14. schemathesis/engine/core.py +1 -1
  15. schemathesis/engine/errors.py +11 -5
  16. schemathesis/engine/events.py +3 -97
  17. schemathesis/engine/phases/stateful/__init__.py +2 -0
  18. schemathesis/engine/phases/stateful/_executor.py +22 -50
  19. schemathesis/engine/phases/unit/__init__.py +1 -0
  20. schemathesis/engine/phases/unit/_executor.py +2 -1
  21. schemathesis/engine/phases/unit/_pool.py +1 -1
  22. schemathesis/engine/recorder.py +29 -23
  23. schemathesis/errors.py +19 -13
  24. schemathesis/generation/coverage.py +4 -4
  25. schemathesis/generation/hypothesis/builder.py +15 -12
  26. schemathesis/generation/stateful/state_machine.py +61 -45
  27. schemathesis/graphql/checks.py +3 -9
  28. schemathesis/openapi/checks.py +8 -33
  29. schemathesis/schemas.py +34 -14
  30. schemathesis/specs/graphql/schemas.py +16 -15
  31. schemathesis/specs/openapi/checks.py +50 -27
  32. schemathesis/specs/openapi/expressions/__init__.py +11 -15
  33. schemathesis/specs/openapi/expressions/nodes.py +20 -20
  34. schemathesis/specs/openapi/links.py +139 -118
  35. schemathesis/specs/openapi/patterns.py +170 -2
  36. schemathesis/specs/openapi/schemas.py +60 -36
  37. schemathesis/specs/openapi/stateful/__init__.py +185 -113
  38. schemathesis/specs/openapi/stateful/control.py +87 -0
  39. {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/METADATA +1 -1
  40. {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/RECORD +43 -43
  41. schemathesis/specs/openapi/expressions/context.py +0 -14
  42. {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/WHEEL +0 -0
  43. {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/entry_points.txt +0 -0
  44. {schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/licenses/LICENSE +0 -0
@@ -2,7 +2,6 @@ from __future__ import annotations
2
2
 
3
3
  import base64
4
4
  import time
5
- import uuid
6
5
  from dataclasses import dataclass
7
6
  from typing import TYPE_CHECKING, Iterator, cast
8
7
 
@@ -14,6 +13,8 @@ from schemathesis.generation.case import Case
14
13
  if TYPE_CHECKING:
15
14
  import requests
16
15
 
16
+ from schemathesis.generation.stateful.state_machine import Transition
17
+
17
18
 
18
19
  @dataclass
19
20
  class ScenarioRecorder:
@@ -22,7 +23,6 @@ class ScenarioRecorder:
22
23
  Records test cases, their hierarchy, API interactions, and results of checks performed during execution.
23
24
  """
24
25
 
25
- id: uuid.UUID
26
26
  # Human-readable label
27
27
  label: str
28
28
 
@@ -33,18 +33,17 @@ class ScenarioRecorder:
33
33
  # Network interactions by test case ID
34
34
  interactions: dict[str, Interaction]
35
35
 
36
- __slots__ = ("id", "label", "status", "roots", "cases", "checks", "interactions")
36
+ __slots__ = ("label", "status", "roots", "cases", "checks", "interactions")
37
37
 
38
38
  def __init__(self, *, label: str) -> None:
39
- self.id = uuid.uuid4()
40
39
  self.label = label
41
40
  self.cases = {}
42
41
  self.checks = {}
43
42
  self.interactions = {}
44
43
 
45
- def record_case(self, *, parent_id: str | None, case: Case) -> None:
44
+ def record_case(self, *, parent_id: str | None, transition: Transition | None, case: Case) -> None:
46
45
  """Record a test case and its relationship to a parent, if applicable."""
47
- self.cases[case.id] = CaseNode(value=case, parent_id=parent_id)
46
+ self.cases[case.id] = CaseNode(value=case, parent_id=parent_id, transition=transition)
48
47
 
49
48
  def record_response(self, *, case_id: str, response: Response) -> None:
50
49
  """Record the API response for a given test case."""
@@ -94,30 +93,34 @@ class ScenarioRecorder:
94
93
  return None
95
94
 
96
95
  def find_related(self, *, case_id: str) -> Iterator[Case]:
97
- """Iterate over all ancestors and their children for a given case."""
98
- current_id = case_id
99
- seen = {current_id}
96
+ """Iterate over all cases in the tree, starting from the root."""
97
+ seen = {case_id}
100
98
 
99
+ # First, find the root by going up
100
+ current_id = case_id
101
101
  while True:
102
102
  current_node = self.cases.get(current_id)
103
103
  if current_node is None or current_node.parent_id is None:
104
+ root_id = current_id
104
105
  break
106
+ current_id = current_node.parent_id
105
107
 
106
- # Get all children of the parent (siblings of the current case)
107
- parent_id = current_node.parent_id
108
- for case_id, maybe_child in self.cases.items():
109
- # If this case has the same parent and we haven't seen it yet
110
- if parent_id == maybe_child.parent_id and case_id not in seen:
108
+ # Then traverse the whole tree from root
109
+ def traverse(node_id: str) -> Iterator[Case]:
110
+ # Get all children
111
+ for case_id, node in self.cases.items():
112
+ if node.parent_id == node_id and case_id not in seen:
111
113
  seen.add(case_id)
112
- yield maybe_child.value
114
+ yield node.value
115
+ # Recurse into children
116
+ yield from traverse(case_id)
113
117
 
114
- # Move up to the parent
115
- current_id = parent_id
116
- if current_id not in seen:
117
- seen.add(current_id)
118
- parent_node = self.cases.get(current_id)
119
- if parent_node:
120
- yield parent_node.value
118
+ # Start traversal from root
119
+ root_node = self.cases.get(root_id)
120
+ if root_node and root_id not in seen:
121
+ seen.add(root_id)
122
+ yield root_node.value
123
+ yield from traverse(root_id)
121
124
 
122
125
  def find_response(self, *, case_id: str) -> Response | None:
123
126
  """Retrieve the API response for a given test case, if available."""
@@ -133,8 +136,11 @@ class CaseNode:
133
136
 
134
137
  value: Case
135
138
  parent_id: str | None
139
+ # Transition may be absent if `parent_id` is present for cases when a case is derived inside a check
140
+ # and outside of the implemented transition logic (e.g. Open API links)
141
+ transition: Transition | None
136
142
 
137
- __slots__ = ("value", "parent_id")
143
+ __slots__ = ("value", "parent_id", "transition")
138
144
 
139
145
 
140
146
  @dataclass
schemathesis/errors.py CHANGED
@@ -1,29 +1,35 @@
1
1
  """Public Schemathesis errors."""
2
2
 
3
- from schemathesis.core.errors import IncorrectUsage as IncorrectUsage
4
- from schemathesis.core.errors import InternalError as InternalError
5
- from schemathesis.core.errors import InvalidHeadersExample as InvalidHeadersExample
6
- from schemathesis.core.errors import InvalidRateLimit as InvalidRateLimit
7
- from schemathesis.core.errors import InvalidRegexPattern as InvalidRegexPattern
8
- from schemathesis.core.errors import InvalidRegexType as InvalidRegexType
9
- from schemathesis.core.errors import InvalidSchema as InvalidSchema
10
- from schemathesis.core.errors import LoaderError as LoaderError
11
- from schemathesis.core.errors import OperationNotFound as OperationNotFound
12
- from schemathesis.core.errors import SchemathesisError as SchemathesisError
13
- from schemathesis.core.errors import SerializationError as SerializationError
14
- from schemathesis.core.errors import SerializationNotPossible as SerializationNotPossible
15
- from schemathesis.core.errors import UnboundPrefix as UnboundPrefix
3
+ from schemathesis.core.errors import (
4
+ IncorrectUsage,
5
+ InternalError,
6
+ InvalidHeadersExample,
7
+ InvalidLinkDefinition,
8
+ InvalidRateLimit,
9
+ InvalidRegexPattern,
10
+ InvalidRegexType,
11
+ InvalidSchema,
12
+ LoaderError,
13
+ NoLinksFound,
14
+ OperationNotFound,
15
+ SchemathesisError,
16
+ SerializationError,
17
+ SerializationNotPossible,
18
+ UnboundPrefix,
19
+ )
16
20
 
17
21
  __all__ = [
18
22
  "IncorrectUsage",
19
23
  "InternalError",
20
24
  "InvalidHeadersExample",
25
+ "InvalidLinkDefinition",
21
26
  "InvalidRateLimit",
22
27
  "InvalidRegexPattern",
23
28
  "InvalidRegexType",
24
29
  "InvalidSchema",
25
30
  "LoaderError",
26
31
  "OperationNotFound",
32
+ "NoLinksFound",
27
33
  "SchemathesisError",
28
34
  "SerializationError",
29
35
  "SerializationNotPossible",
@@ -194,6 +194,10 @@ class CoverageContext:
194
194
  re.compile(pattern)
195
195
  except re.error:
196
196
  raise Unsatisfiable from None
197
+ if "minLength" in schema or "maxLength" in schema:
198
+ min_length = schema.get("minLength")
199
+ max_length = schema.get("maxLength")
200
+ pattern = update_quantifier(pattern, min_length, max_length)
197
201
  return cached_draw(st.from_regex(pattern))
198
202
  if (keys == ["items", "type"] or keys == ["items", "minItems", "type"]) and isinstance(schema["items"], dict):
199
203
  items = schema["items"]
@@ -514,11 +518,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
514
518
  # Default positive value
515
519
  yield PositiveValue(ctx.generate_from_schema(schema), description="Valid string")
516
520
  elif "pattern" in schema:
517
- # Without merging `maxLength` & `minLength` into a regex it is problematic
518
- # to generate a valid value as the unredlying machinery will resort to filtering
519
- # and it is unlikely that it will generate a string of that length
520
521
  yield PositiveValue(ctx.generate_from_schema(schema), description="Valid string")
521
- return
522
522
 
523
523
  seen = set()
524
524
 
@@ -386,19 +386,22 @@ def _iter_coverage_cases(
386
386
  container = template["query"]
387
387
  for parameter in operation.query:
388
388
  instant = Instant()
389
- value = container[parameter.name]
390
- yield operation.Case(
391
- **{**template, "query": {**container, parameter.name: [value, value]}},
392
- meta=CaseMetadata(
393
- generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
394
- components={},
395
- phase=PhaseInfo.coverage(
396
- description=f"Duplicate `{parameter.name}` query parameter",
397
- parameter=parameter.name,
398
- parameter_location="query",
389
+ # Could be absent if value schema can't be negated
390
+ # I.e. contains just `default` value without any other keywords
391
+ value = container.get(parameter.name, NOT_SET)
392
+ if value is not NOT_SET:
393
+ yield operation.Case(
394
+ **{**template, "query": {**container, parameter.name: [value, value]}},
395
+ meta=CaseMetadata(
396
+ generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
397
+ components={},
398
+ phase=PhaseInfo.coverage(
399
+ description=f"Duplicate `{parameter.name}` query parameter",
400
+ parameter=parameter.name,
401
+ parameter_location="query",
402
+ ),
399
403
  ),
400
- ),
401
- )
404
+ )
402
405
  # Generate missing required parameters
403
406
  for parameter in operation.iter_parameters():
404
407
  if parameter.is_required and parameter.location != "path":
@@ -10,7 +10,8 @@ from hypothesis.errors import InvalidDefinition
10
10
  from hypothesis.stateful import RuleBasedStateMachine
11
11
 
12
12
  from schemathesis.checks import CheckFunction
13
- from schemathesis.core.errors import IncorrectUsage
13
+ from schemathesis.core.errors import NoLinksFound
14
+ from schemathesis.core.result import Result
14
15
  from schemathesis.core.transport import Response
15
16
  from schemathesis.generation.case import Case
16
17
 
@@ -18,30 +19,64 @@ if TYPE_CHECKING:
18
19
  import hypothesis
19
20
  from requests.structures import CaseInsensitiveDict
20
21
 
21
- from schemathesis.schemas import APIOperation, BaseSchema
22
+ from schemathesis.schemas import BaseSchema
22
23
 
23
24
 
24
- NO_LINKS_ERROR_MESSAGE = (
25
- "Stateful testing requires at least one OpenAPI link in the schema, but no links detected. "
26
- "Please add OpenAPI links to enable stateful testing or use stateless tests instead. \n"
27
- "See https://schemathesis.readthedocs.io/en/stable/stateful.html#how-to-specify-connections for more information."
28
- )
29
-
25
+ DEFAULT_STATEFUL_STEP_COUNT = 6
30
26
  DEFAULT_STATE_MACHINE_SETTINGS = hypothesis.settings(
31
27
  phases=[hypothesis.Phase.generate],
32
28
  deadline=None,
33
- stateful_step_count=6,
29
+ stateful_step_count=DEFAULT_STATEFUL_STEP_COUNT,
34
30
  suppress_health_check=list(hypothesis.HealthCheck),
35
31
  )
36
32
 
37
33
 
38
34
  @dataclass
39
- class StepResult:
35
+ class StepInput:
36
+ """Input for a single state machine step."""
37
+
38
+ case: Case
39
+ transition: Transition | None # None for initial steps
40
+
41
+ __slots__ = ("case", "transition")
42
+
43
+ @classmethod
44
+ def initial(cls, case: Case) -> StepInput:
45
+ return cls(case=case, transition=None)
46
+
47
+
48
+ @dataclass
49
+ class Transition:
50
+ """Data about transition execution."""
51
+
52
+ # ID of the transition (e.g. link name)
53
+ id: str
54
+ parent_id: str
55
+ parameters: dict[str, dict[str, ExtractedParam]]
56
+ request_body: ExtractedParam | None
57
+
58
+ __slots__ = ("id", "parent_id", "parameters", "request_body")
59
+
60
+
61
+ @dataclass
62
+ class ExtractedParam:
63
+ """Result of parameter extraction."""
64
+
65
+ definition: Any
66
+ value: Result[Any, Exception]
67
+
68
+ __slots__ = ("definition", "value")
69
+
70
+
71
+ @dataclass
72
+ class StepOutput:
40
73
  """Output from a single transition of a state machine."""
41
74
 
42
75
  response: Response
43
76
  case: Case
44
77
 
78
+ __slots__ = ("response", "case")
79
+
45
80
 
46
81
  def _normalize_name(name: str) -> str:
47
82
  return re.sub(r"\W|^(?=\d)", "_", name).replace("__", "_")
@@ -64,7 +99,11 @@ class APIStateMachine(RuleBasedStateMachine):
64
99
  super().__init__() # type: ignore
65
100
  except InvalidDefinition as exc:
66
101
  if "defines no rules" in str(exc):
67
- raise IncorrectUsage(NO_LINKS_ERROR_MESSAGE) from None
102
+ if not self.schema.statistic.links.total:
103
+ message = "Schema contains no link definitions required for stateful testing"
104
+ else:
105
+ message = "All link definitions required for stateful testing are excluded by filters"
106
+ raise NoLinksFound(message) from None
68
107
  raise
69
108
  self.setup()
70
109
 
@@ -89,10 +128,10 @@ class APIStateMachine(RuleBasedStateMachine):
89
128
  target = _normalize_name(target)
90
129
  return super()._new_name(target) # type: ignore
91
130
 
92
- def _get_target_for_result(self, result: StepResult) -> str | None:
131
+ def _get_target_for_result(self, result: StepOutput) -> str | None:
93
132
  raise NotImplementedError
94
133
 
95
- def _add_result_to_targets(self, targets: tuple[str, ...], result: StepResult | None) -> None:
134
+ def _add_result_to_targets(self, targets: tuple[str, ...], result: StepOutput | None) -> None:
96
135
  if result is None:
97
136
  return
98
137
  target = self._get_target_for_result(result)
@@ -115,19 +154,11 @@ class APIStateMachine(RuleBasedStateMachine):
115
154
  # To provide the return type in the rendered documentation
116
155
  teardown.__doc__ = RuleBasedStateMachine.teardown.__doc__
117
156
 
118
- def transform(self, result: StepResult, direction: Direction, case: Case) -> Case:
119
- raise NotImplementedError
120
-
121
- def _step(self, case: Case, previous: StepResult | None = None, link: Direction | None = None) -> StepResult | None:
122
- # This method is a proxy that is used under the hood during the state machine initialization.
123
- # The whole point of having it is to make it possible to override `step`; otherwise, custom "step" is ignored.
124
- # It happens because, at the point of initialization, the final class is not yet created.
157
+ def _step(self, input: StepInput) -> StepOutput | None:
125
158
  __tracebackhide__ = True
126
- if previous is not None and link is not None:
127
- return self.step(case, (previous, link))
128
- return self.step(case, None)
159
+ return self.step(input)
129
160
 
130
- def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult:
161
+ def step(self, input: StepInput) -> StepOutput:
131
162
  """A single state machine step.
132
163
 
133
164
  :param Case case: Generated test case data that should be sent in an API call to the tested API operation.
@@ -137,15 +168,12 @@ class APIStateMachine(RuleBasedStateMachine):
137
168
  It is the most high-level point to extend the testing process. You probably don't need it in most cases.
138
169
  """
139
170
  __tracebackhide__ = True
140
- if previous is not None:
141
- result, direction = previous
142
- case = self.transform(result, direction, case)
143
- self.before_call(case)
144
- kwargs = self.get_call_kwargs(case)
145
- response = self.call(case, **kwargs)
146
- self.after_call(response, case)
147
- self.validate_response(response, case)
148
- return self.store_result(response, case)
171
+ self.before_call(input.case)
172
+ kwargs = self.get_call_kwargs(input.case)
173
+ response = self.call(input.case, **kwargs)
174
+ self.after_call(response, input.case)
175
+ self.validate_response(response, input.case)
176
+ return StepOutput(response, input.case)
149
177
 
150
178
  def before_call(self, case: Case) -> None:
151
179
  """Hook method for modifying the case data before making a request.
@@ -271,15 +299,3 @@ class APIStateMachine(RuleBasedStateMachine):
271
299
  """
272
300
  __tracebackhide__ = True
273
301
  case.validate_response(response, additional_checks=additional_checks)
274
-
275
- def store_result(self, response: Response, case: Case) -> StepResult:
276
- return StepResult(response, case)
277
-
278
-
279
- class Direction:
280
- name: str
281
- status_code: str
282
- operation: APIOperation
283
-
284
- def set_data(self, case: Case, **kwargs: Any) -> None:
285
- raise NotImplementedError
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
11
11
  class UnexpectedGraphQLResponse(Failure):
12
12
  """GraphQL response is not a JSON object."""
13
13
 
14
- __slots__ = ("operation", "type_name", "title", "message", "code", "case_id", "severity")
14
+ __slots__ = ("operation", "type_name", "title", "message", "case_id", "severity")
15
15
 
16
16
  def __init__(
17
17
  self,
@@ -20,14 +20,12 @@ class UnexpectedGraphQLResponse(Failure):
20
20
  type_name: str,
21
21
  title: str = "Unexpected GraphQL Response",
22
22
  message: str,
23
- code: str = "graphql_unexpected_response",
24
23
  case_id: str | None = None,
25
24
  ) -> None:
26
25
  self.operation = operation
27
26
  self.type_name = type_name
28
27
  self.title = title
29
28
  self.message = message
30
- self.code = code
31
29
  self.case_id = case_id
32
30
  self.severity = Severity.MEDIUM
33
31
 
@@ -39,7 +37,7 @@ class UnexpectedGraphQLResponse(Failure):
39
37
  class GraphQLClientError(Failure):
40
38
  """GraphQL query has not been executed."""
41
39
 
42
- __slots__ = ("operation", "errors", "title", "message", "code", "case_id", "_unique_key_cache", "severity")
40
+ __slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
43
41
 
44
42
  def __init__(
45
43
  self,
@@ -48,14 +46,12 @@ class GraphQLClientError(Failure):
48
46
  message: str,
49
47
  errors: list[GraphQLFormattedError],
50
48
  title: str = "GraphQL client error",
51
- code: str = "graphql_client_error",
52
49
  case_id: str | None = None,
53
50
  ) -> None:
54
51
  self.operation = operation
55
52
  self.errors = errors
56
53
  self.title = title
57
54
  self.message = message
58
- self.code = code
59
55
  self.case_id = case_id
60
56
  self._unique_key_cache: str | None = None
61
57
  self.severity = Severity.MEDIUM
@@ -70,7 +66,7 @@ class GraphQLClientError(Failure):
70
66
  class GraphQLServerError(Failure):
71
67
  """GraphQL response indicates at least one server error."""
72
68
 
73
- __slots__ = ("operation", "errors", "title", "message", "code", "case_id", "_unique_key_cache", "severity")
69
+ __slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
74
70
 
75
71
  def __init__(
76
72
  self,
@@ -79,14 +75,12 @@ class GraphQLServerError(Failure):
79
75
  message: str,
80
76
  errors: list[GraphQLFormattedError],
81
77
  title: str = "GraphQL server error",
82
- code: str = "graphql_server_error",
83
78
  case_id: str | None = None,
84
79
  ) -> None:
85
80
  self.operation = operation
86
81
  self.errors = errors
87
82
  self.title = title
88
83
  self.message = message
89
- self.code = code
90
84
  self.case_id = case_id
91
85
  self._unique_key_cache: str | None = None
92
86
  self.severity = Severity.CRITICAL