schemathesis 3.15.4__py3-none-any.whl → 4.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. schemathesis/__init__.py +53 -25
  2. schemathesis/auths.py +507 -0
  3. schemathesis/checks.py +190 -25
  4. schemathesis/cli/__init__.py +27 -1219
  5. schemathesis/cli/__main__.py +4 -0
  6. schemathesis/cli/commands/__init__.py +133 -0
  7. schemathesis/cli/commands/data.py +10 -0
  8. schemathesis/cli/commands/run/__init__.py +602 -0
  9. schemathesis/cli/commands/run/context.py +228 -0
  10. schemathesis/cli/commands/run/events.py +60 -0
  11. schemathesis/cli/commands/run/executor.py +157 -0
  12. schemathesis/cli/commands/run/filters.py +53 -0
  13. schemathesis/cli/commands/run/handlers/__init__.py +46 -0
  14. schemathesis/cli/commands/run/handlers/base.py +45 -0
  15. schemathesis/cli/commands/run/handlers/cassettes.py +464 -0
  16. schemathesis/cli/commands/run/handlers/junitxml.py +60 -0
  17. schemathesis/cli/commands/run/handlers/output.py +1750 -0
  18. schemathesis/cli/commands/run/loaders.py +118 -0
  19. schemathesis/cli/commands/run/validation.py +256 -0
  20. schemathesis/cli/constants.py +5 -0
  21. schemathesis/cli/core.py +19 -0
  22. schemathesis/cli/ext/fs.py +16 -0
  23. schemathesis/cli/ext/groups.py +203 -0
  24. schemathesis/cli/ext/options.py +81 -0
  25. schemathesis/config/__init__.py +202 -0
  26. schemathesis/config/_auth.py +51 -0
  27. schemathesis/config/_checks.py +268 -0
  28. schemathesis/config/_diff_base.py +101 -0
  29. schemathesis/config/_env.py +21 -0
  30. schemathesis/config/_error.py +163 -0
  31. schemathesis/config/_generation.py +157 -0
  32. schemathesis/config/_health_check.py +24 -0
  33. schemathesis/config/_operations.py +335 -0
  34. schemathesis/config/_output.py +171 -0
  35. schemathesis/config/_parameters.py +19 -0
  36. schemathesis/config/_phases.py +253 -0
  37. schemathesis/config/_projects.py +543 -0
  38. schemathesis/config/_rate_limit.py +17 -0
  39. schemathesis/config/_report.py +120 -0
  40. schemathesis/config/_validator.py +9 -0
  41. schemathesis/config/_warnings.py +89 -0
  42. schemathesis/config/schema.json +975 -0
  43. schemathesis/core/__init__.py +72 -0
  44. schemathesis/core/adapter.py +34 -0
  45. schemathesis/core/compat.py +32 -0
  46. schemathesis/core/control.py +2 -0
  47. schemathesis/core/curl.py +100 -0
  48. schemathesis/core/deserialization.py +210 -0
  49. schemathesis/core/errors.py +588 -0
  50. schemathesis/core/failures.py +316 -0
  51. schemathesis/core/fs.py +19 -0
  52. schemathesis/core/hooks.py +20 -0
  53. schemathesis/core/jsonschema/__init__.py +13 -0
  54. schemathesis/core/jsonschema/bundler.py +183 -0
  55. schemathesis/core/jsonschema/keywords.py +40 -0
  56. schemathesis/core/jsonschema/references.py +222 -0
  57. schemathesis/core/jsonschema/types.py +41 -0
  58. schemathesis/core/lazy_import.py +15 -0
  59. schemathesis/core/loaders.py +107 -0
  60. schemathesis/core/marks.py +66 -0
  61. schemathesis/core/media_types.py +79 -0
  62. schemathesis/core/output/__init__.py +46 -0
  63. schemathesis/core/output/sanitization.py +54 -0
  64. schemathesis/core/parameters.py +45 -0
  65. schemathesis/core/rate_limit.py +60 -0
  66. schemathesis/core/registries.py +34 -0
  67. schemathesis/core/result.py +27 -0
  68. schemathesis/core/schema_analysis.py +17 -0
  69. schemathesis/core/shell.py +203 -0
  70. schemathesis/core/transforms.py +144 -0
  71. schemathesis/core/transport.py +223 -0
  72. schemathesis/core/validation.py +73 -0
  73. schemathesis/core/version.py +7 -0
  74. schemathesis/engine/__init__.py +28 -0
  75. schemathesis/engine/context.py +152 -0
  76. schemathesis/engine/control.py +44 -0
  77. schemathesis/engine/core.py +201 -0
  78. schemathesis/engine/errors.py +446 -0
  79. schemathesis/engine/events.py +284 -0
  80. schemathesis/engine/observations.py +42 -0
  81. schemathesis/engine/phases/__init__.py +108 -0
  82. schemathesis/engine/phases/analysis.py +28 -0
  83. schemathesis/engine/phases/probes.py +172 -0
  84. schemathesis/engine/phases/stateful/__init__.py +68 -0
  85. schemathesis/engine/phases/stateful/_executor.py +364 -0
  86. schemathesis/engine/phases/stateful/context.py +85 -0
  87. schemathesis/engine/phases/unit/__init__.py +220 -0
  88. schemathesis/engine/phases/unit/_executor.py +459 -0
  89. schemathesis/engine/phases/unit/_pool.py +82 -0
  90. schemathesis/engine/recorder.py +254 -0
  91. schemathesis/errors.py +47 -0
  92. schemathesis/filters.py +395 -0
  93. schemathesis/generation/__init__.py +25 -0
  94. schemathesis/generation/case.py +478 -0
  95. schemathesis/generation/coverage.py +1528 -0
  96. schemathesis/generation/hypothesis/__init__.py +121 -0
  97. schemathesis/generation/hypothesis/builder.py +992 -0
  98. schemathesis/generation/hypothesis/examples.py +56 -0
  99. schemathesis/generation/hypothesis/given.py +66 -0
  100. schemathesis/generation/hypothesis/reporting.py +285 -0
  101. schemathesis/generation/meta.py +227 -0
  102. schemathesis/generation/metrics.py +93 -0
  103. schemathesis/generation/modes.py +20 -0
  104. schemathesis/generation/overrides.py +127 -0
  105. schemathesis/generation/stateful/__init__.py +37 -0
  106. schemathesis/generation/stateful/state_machine.py +294 -0
  107. schemathesis/graphql/__init__.py +15 -0
  108. schemathesis/graphql/checks.py +109 -0
  109. schemathesis/graphql/loaders.py +285 -0
  110. schemathesis/hooks.py +270 -91
  111. schemathesis/openapi/__init__.py +13 -0
  112. schemathesis/openapi/checks.py +467 -0
  113. schemathesis/openapi/generation/__init__.py +0 -0
  114. schemathesis/openapi/generation/filters.py +72 -0
  115. schemathesis/openapi/loaders.py +315 -0
  116. schemathesis/pytest/__init__.py +5 -0
  117. schemathesis/pytest/control_flow.py +7 -0
  118. schemathesis/pytest/lazy.py +341 -0
  119. schemathesis/pytest/loaders.py +36 -0
  120. schemathesis/pytest/plugin.py +357 -0
  121. schemathesis/python/__init__.py +0 -0
  122. schemathesis/python/asgi.py +12 -0
  123. schemathesis/python/wsgi.py +12 -0
  124. schemathesis/schemas.py +682 -257
  125. schemathesis/specs/graphql/__init__.py +0 -1
  126. schemathesis/specs/graphql/nodes.py +26 -2
  127. schemathesis/specs/graphql/scalars.py +77 -12
  128. schemathesis/specs/graphql/schemas.py +367 -148
  129. schemathesis/specs/graphql/validation.py +33 -0
  130. schemathesis/specs/openapi/__init__.py +9 -1
  131. schemathesis/specs/openapi/_hypothesis.py +555 -318
  132. schemathesis/specs/openapi/adapter/__init__.py +10 -0
  133. schemathesis/specs/openapi/adapter/parameters.py +729 -0
  134. schemathesis/specs/openapi/adapter/protocol.py +59 -0
  135. schemathesis/specs/openapi/adapter/references.py +19 -0
  136. schemathesis/specs/openapi/adapter/responses.py +368 -0
  137. schemathesis/specs/openapi/adapter/security.py +144 -0
  138. schemathesis/specs/openapi/adapter/v2.py +30 -0
  139. schemathesis/specs/openapi/adapter/v3_0.py +30 -0
  140. schemathesis/specs/openapi/adapter/v3_1.py +30 -0
  141. schemathesis/specs/openapi/analysis.py +96 -0
  142. schemathesis/specs/openapi/checks.py +748 -82
  143. schemathesis/specs/openapi/converter.py +176 -37
  144. schemathesis/specs/openapi/definitions.py +599 -4
  145. schemathesis/specs/openapi/examples.py +581 -165
  146. schemathesis/specs/openapi/expressions/__init__.py +52 -5
  147. schemathesis/specs/openapi/expressions/extractors.py +25 -0
  148. schemathesis/specs/openapi/expressions/lexer.py +34 -31
  149. schemathesis/specs/openapi/expressions/nodes.py +97 -46
  150. schemathesis/specs/openapi/expressions/parser.py +35 -13
  151. schemathesis/specs/openapi/formats.py +122 -0
  152. schemathesis/specs/openapi/media_types.py +75 -0
  153. schemathesis/specs/openapi/negative/__init__.py +93 -73
  154. schemathesis/specs/openapi/negative/mutations.py +294 -103
  155. schemathesis/specs/openapi/negative/utils.py +0 -9
  156. schemathesis/specs/openapi/patterns.py +458 -0
  157. schemathesis/specs/openapi/references.py +60 -81
  158. schemathesis/specs/openapi/schemas.py +647 -666
  159. schemathesis/specs/openapi/serialization.py +53 -30
  160. schemathesis/specs/openapi/stateful/__init__.py +403 -68
  161. schemathesis/specs/openapi/stateful/control.py +87 -0
  162. schemathesis/specs/openapi/stateful/dependencies/__init__.py +232 -0
  163. schemathesis/specs/openapi/stateful/dependencies/inputs.py +428 -0
  164. schemathesis/specs/openapi/stateful/dependencies/models.py +341 -0
  165. schemathesis/specs/openapi/stateful/dependencies/naming.py +491 -0
  166. schemathesis/specs/openapi/stateful/dependencies/outputs.py +34 -0
  167. schemathesis/specs/openapi/stateful/dependencies/resources.py +339 -0
  168. schemathesis/specs/openapi/stateful/dependencies/schemas.py +447 -0
  169. schemathesis/specs/openapi/stateful/inference.py +254 -0
  170. schemathesis/specs/openapi/stateful/links.py +219 -78
  171. schemathesis/specs/openapi/types/__init__.py +3 -0
  172. schemathesis/specs/openapi/types/common.py +23 -0
  173. schemathesis/specs/openapi/types/v2.py +129 -0
  174. schemathesis/specs/openapi/types/v3.py +134 -0
  175. schemathesis/specs/openapi/utils.py +7 -6
  176. schemathesis/specs/openapi/warnings.py +75 -0
  177. schemathesis/transport/__init__.py +224 -0
  178. schemathesis/transport/asgi.py +26 -0
  179. schemathesis/transport/prepare.py +126 -0
  180. schemathesis/transport/requests.py +278 -0
  181. schemathesis/transport/serialization.py +329 -0
  182. schemathesis/transport/wsgi.py +175 -0
  183. schemathesis-4.4.2.dist-info/METADATA +213 -0
  184. schemathesis-4.4.2.dist-info/RECORD +192 -0
  185. {schemathesis-3.15.4.dist-info → schemathesis-4.4.2.dist-info}/WHEEL +1 -1
  186. schemathesis-4.4.2.dist-info/entry_points.txt +6 -0
  187. {schemathesis-3.15.4.dist-info → schemathesis-4.4.2.dist-info/licenses}/LICENSE +1 -1
  188. schemathesis/_compat.py +0 -57
  189. schemathesis/_hypothesis.py +0 -123
  190. schemathesis/auth.py +0 -214
  191. schemathesis/cli/callbacks.py +0 -240
  192. schemathesis/cli/cassettes.py +0 -351
  193. schemathesis/cli/context.py +0 -38
  194. schemathesis/cli/debug.py +0 -21
  195. schemathesis/cli/handlers.py +0 -11
  196. schemathesis/cli/junitxml.py +0 -41
  197. schemathesis/cli/options.py +0 -70
  198. schemathesis/cli/output/__init__.py +0 -1
  199. schemathesis/cli/output/default.py +0 -521
  200. schemathesis/cli/output/short.py +0 -40
  201. schemathesis/constants.py +0 -88
  202. schemathesis/exceptions.py +0 -257
  203. schemathesis/extra/_aiohttp.py +0 -27
  204. schemathesis/extra/_flask.py +0 -10
  205. schemathesis/extra/_server.py +0 -16
  206. schemathesis/extra/pytest_plugin.py +0 -251
  207. schemathesis/failures.py +0 -145
  208. schemathesis/fixups/__init__.py +0 -29
  209. schemathesis/fixups/fast_api.py +0 -30
  210. schemathesis/graphql.py +0 -5
  211. schemathesis/internal.py +0 -6
  212. schemathesis/lazy.py +0 -301
  213. schemathesis/models.py +0 -1113
  214. schemathesis/parameters.py +0 -91
  215. schemathesis/runner/__init__.py +0 -470
  216. schemathesis/runner/events.py +0 -242
  217. schemathesis/runner/impl/__init__.py +0 -3
  218. schemathesis/runner/impl/core.py +0 -791
  219. schemathesis/runner/impl/solo.py +0 -85
  220. schemathesis/runner/impl/threadpool.py +0 -367
  221. schemathesis/runner/serialization.py +0 -206
  222. schemathesis/serializers.py +0 -253
  223. schemathesis/service/__init__.py +0 -18
  224. schemathesis/service/auth.py +0 -10
  225. schemathesis/service/client.py +0 -62
  226. schemathesis/service/constants.py +0 -25
  227. schemathesis/service/events.py +0 -39
  228. schemathesis/service/handler.py +0 -46
  229. schemathesis/service/hosts.py +0 -74
  230. schemathesis/service/metadata.py +0 -42
  231. schemathesis/service/models.py +0 -21
  232. schemathesis/service/serialization.py +0 -184
  233. schemathesis/service/worker.py +0 -39
  234. schemathesis/specs/graphql/loaders.py +0 -215
  235. schemathesis/specs/openapi/constants.py +0 -7
  236. schemathesis/specs/openapi/expressions/context.py +0 -12
  237. schemathesis/specs/openapi/expressions/pointers.py +0 -29
  238. schemathesis/specs/openapi/filters.py +0 -44
  239. schemathesis/specs/openapi/links.py +0 -303
  240. schemathesis/specs/openapi/loaders.py +0 -453
  241. schemathesis/specs/openapi/parameters.py +0 -430
  242. schemathesis/specs/openapi/security.py +0 -129
  243. schemathesis/specs/openapi/validation.py +0 -24
  244. schemathesis/stateful.py +0 -358
  245. schemathesis/targets.py +0 -32
  246. schemathesis/types.py +0 -38
  247. schemathesis/utils.py +0 -475
  248. schemathesis-3.15.4.dist-info/METADATA +0 -202
  249. schemathesis-3.15.4.dist-info/RECORD +0 -99
  250. schemathesis-3.15.4.dist-info/entry_points.txt +0 -7
  251. /schemathesis/{extra → cli/ext}/__init__.py +0 -0
@@ -1,791 +0,0 @@
1
- # pylint: disable=too-many-statements,too-many-branches
2
- import logging
3
- import threading
4
- import time
5
- import uuid
6
- from contextlib import contextmanager
7
- from types import TracebackType
8
- from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Type, Union, cast
9
- from warnings import WarningMessage, catch_warnings
10
-
11
- import attr
12
- import hypothesis
13
- import requests
14
- from _pytest.logging import LogCaptureHandler, catching_logs
15
- from hypothesis.errors import HypothesisException, InvalidArgument
16
- from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
17
- from requests.auth import HTTPDigestAuth, _basic_auth_str
18
-
19
- from ... import failures, hooks
20
- from ...auth import unregister as unregister_auth
21
- from ...constants import (
22
- DEFAULT_STATEFUL_RECURSION_LIMIT,
23
- RECURSIVE_REFERENCE_ERROR_MESSAGE,
24
- USER_AGENT,
25
- DataGenerationMethod,
26
- )
27
- from ...exceptions import (
28
- CheckFailed,
29
- DeadlineExceeded,
30
- InvalidRegularExpression,
31
- InvalidSchema,
32
- NonCheckError,
33
- SkipTest,
34
- get_grouped_exception,
35
- )
36
- from ...hooks import HookContext, get_all_by_name
37
- from ...models import APIOperation, Case, Check, CheckFunction, Status, TestResult, TestResultSet
38
- from ...runner import events
39
- from ...schemas import BaseSchema
40
- from ...stateful import Feedback, Stateful
41
- from ...targets import Target, TargetContext
42
- from ...types import RawAuth, RequestCert
43
- from ...utils import (
44
- GenericResponse,
45
- Ok,
46
- WSGIResponse,
47
- capture_hypothesis_output,
48
- copy_response,
49
- format_exception,
50
- maybe_set_assertion_message,
51
- )
52
- from ..serialization import SerializedTestResult
53
-
54
-
55
- @attr.s # pragma: no mutate
56
- class BaseRunner:
57
- schema: BaseSchema = attr.ib() # pragma: no mutate
58
- checks: Iterable[CheckFunction] = attr.ib() # pragma: no mutate
59
- max_response_time: Optional[int] = attr.ib() # pragma: no mutate
60
- targets: Iterable[Target] = attr.ib() # pragma: no mutate
61
- hypothesis_settings: hypothesis.settings = attr.ib() # pragma: no mutate
62
- auth: Optional[RawAuth] = attr.ib(default=None) # pragma: no mutate
63
- auth_type: Optional[str] = attr.ib(default=None) # pragma: no mutate
64
- headers: Optional[Dict[str, Any]] = attr.ib(default=None) # pragma: no mutate
65
- request_timeout: Optional[int] = attr.ib(default=None) # pragma: no mutate
66
- store_interactions: bool = attr.ib(default=False) # pragma: no mutate
67
- seed: Optional[int] = attr.ib(default=None) # pragma: no mutate
68
- exit_first: bool = attr.ib(default=False) # pragma: no mutate
69
- dry_run: bool = attr.ib(default=False) # pragma: no mutate
70
- stateful: Optional[Stateful] = attr.ib(default=None) # pragma: no mutate
71
- stateful_recursion_limit: int = attr.ib(default=DEFAULT_STATEFUL_RECURSION_LIMIT) # pragma: no mutate
72
- count_operations: bool = attr.ib(default=True) # pragma: no mutate
73
-
74
- def execute(self) -> "EventStream":
75
- """Common logic for all runners."""
76
- event = threading.Event()
77
- return EventStream(self._generate_events(event), event)
78
-
79
- def _generate_events(self, stop_event: threading.Event) -> Generator[events.ExecutionEvent, None, None]:
80
- # If auth is explicitly provided, then the global provider is ignored
81
- if self.auth is not None:
82
- unregister_auth()
83
- results = TestResultSet()
84
-
85
- initialized = events.Initialized.from_schema(schema=self.schema, count_operations=self.count_operations)
86
-
87
- def _finish() -> events.Finished:
88
- return events.Finished.from_results(results=results, running_time=time.monotonic() - initialized.start_time)
89
-
90
- if stop_event.is_set():
91
- yield _finish()
92
- return
93
-
94
- yield initialized
95
-
96
- if stop_event.is_set():
97
- yield _finish()
98
- return
99
-
100
- try:
101
- for event in self._execute(results, stop_event):
102
- yield event
103
- except KeyboardInterrupt:
104
- yield events.Interrupted()
105
-
106
- yield _finish()
107
-
108
- def _should_stop(self, event: events.ExecutionEvent) -> bool:
109
- return (
110
- self.exit_first
111
- and isinstance(event, events.AfterExecution)
112
- and event.status in (Status.error, Status.failure)
113
- )
114
-
115
- def _execute(
116
- self, results: TestResultSet, stop_event: threading.Event
117
- ) -> Generator[events.ExecutionEvent, None, None]:
118
- raise NotImplementedError
119
-
120
- def _run_tests(
121
- self,
122
- maker: Callable,
123
- template: Callable,
124
- settings: hypothesis.settings,
125
- seed: Optional[int],
126
- results: TestResultSet,
127
- recursion_level: int = 0,
128
- **kwargs: Any,
129
- ) -> Generator[events.ExecutionEvent, None, None]:
130
- """Run tests and recursively run additional tests."""
131
- if recursion_level > self.stateful_recursion_limit:
132
- return
133
- for result, data_generation_method in maker(template, settings, seed):
134
- if isinstance(result, Ok):
135
- operation, test = result.ok()
136
- feedback = Feedback(self.stateful, operation)
137
- # Track whether `BeforeExecution` was already emitted.
138
- # Schema error may happen before / after `BeforeExecution`, but it should be emitted only once
139
- # and the `AfterExecution` event should have the same correlation id as previous `BeforeExecution`
140
- before_execution_correlation_id = None
141
- try:
142
- for event in run_test(
143
- operation,
144
- test,
145
- results=results,
146
- feedback=feedback,
147
- recursion_level=recursion_level,
148
- data_generation_method=data_generation_method,
149
- **kwargs,
150
- ):
151
- yield event
152
- if isinstance(event, events.BeforeExecution):
153
- before_execution_correlation_id = event.correlation_id
154
- if isinstance(event, events.Interrupted):
155
- return
156
- # Additional tests, generated via the `feedback` instance
157
- yield from self._run_tests(
158
- feedback.get_stateful_tests,
159
- template,
160
- settings,
161
- seed,
162
- recursion_level=recursion_level + 1,
163
- results=results,
164
- **kwargs,
165
- )
166
- except InvalidSchema as exc:
167
- yield from handle_schema_error(
168
- exc,
169
- results,
170
- data_generation_method,
171
- recursion_level,
172
- before_execution_correlation_id=before_execution_correlation_id,
173
- )
174
- else:
175
- # Schema errors
176
- yield from handle_schema_error(result.err(), results, data_generation_method, recursion_level)
177
-
178
-
179
- @attr.s(slots=True) # pragma: no mutate
180
- class EventStream:
181
- """Schemathesis event stream.
182
-
183
- Provides an API to control the execution flow.
184
- """
185
-
186
- generator: Generator[events.ExecutionEvent, None, None] = attr.ib() # pragma: no mutate
187
- stop_event: threading.Event = attr.ib() # pragma: no mutate
188
-
189
- def __next__(self) -> events.ExecutionEvent:
190
- return next(self.generator)
191
-
192
- def __iter__(self) -> Generator[events.ExecutionEvent, None, None]:
193
- return self.generator
194
-
195
- def stop(self) -> None:
196
- """Stop the event stream.
197
-
198
- Its next value will be the last one (Finished).
199
- """
200
- self.stop_event.set()
201
-
202
- def finish(self) -> events.ExecutionEvent:
203
- """Stop the event stream & return the last event."""
204
- self.stop()
205
- return next(self)
206
-
207
-
208
- def handle_schema_error(
209
- error: InvalidSchema,
210
- results: TestResultSet,
211
- data_generation_method: DataGenerationMethod,
212
- recursion_level: int,
213
- *,
214
- before_execution_correlation_id: Optional[str] = None,
215
- ) -> Generator[events.ExecutionEvent, None, None]:
216
- if error.method is not None:
217
- assert error.path is not None
218
- assert error.full_path is not None
219
- method = error.method.upper()
220
- verbose_name = f"{method} {error.full_path}"
221
- result = TestResult(
222
- method=method,
223
- path=error.full_path,
224
- verbose_name=verbose_name,
225
- data_generation_method=data_generation_method,
226
- )
227
- result.add_error(error)
228
- # It might be already emitted - reuse its correlation id
229
- if before_execution_correlation_id is not None:
230
- correlation_id = before_execution_correlation_id
231
- else:
232
- correlation_id = uuid.uuid4().hex
233
- yield events.BeforeExecution(
234
- method=method,
235
- path=error.full_path,
236
- verbose_name=verbose_name,
237
- relative_path=error.path,
238
- recursion_level=recursion_level,
239
- data_generation_method=data_generation_method,
240
- correlation_id=correlation_id,
241
- )
242
- yield events.AfterExecution(
243
- method=method,
244
- path=error.full_path,
245
- relative_path=error.path,
246
- verbose_name=verbose_name,
247
- status=Status.error,
248
- result=SerializedTestResult.from_test_result(result),
249
- data_generation_method=data_generation_method,
250
- elapsed_time=0.0,
251
- hypothesis_output=[],
252
- correlation_id=correlation_id,
253
- )
254
- results.append(result)
255
- else:
256
- # When there is no `method`, then the schema error may cover multiple operations, and we can't display it in
257
- # the progress bar
258
- results.generic_errors.append(error)
259
-
260
-
261
- def run_test( # pylint: disable=too-many-locals
262
- operation: APIOperation,
263
- test: Callable,
264
- checks: Iterable[CheckFunction],
265
- data_generation_method: DataGenerationMethod,
266
- targets: Iterable[Target],
267
- results: TestResultSet,
268
- headers: Optional[Dict[str, Any]],
269
- recursion_level: int,
270
- **kwargs: Any,
271
- ) -> Generator[events.ExecutionEvent, None, None]:
272
- """A single test run with all error handling needed."""
273
- result = TestResult(
274
- method=operation.method.upper(),
275
- path=operation.full_path,
276
- verbose_name=operation.verbose_name,
277
- overridden_headers=headers,
278
- data_generation_method=data_generation_method,
279
- )
280
- # To simplify connecting `before` and `after` events in external systems
281
- correlation_id = uuid.uuid4().hex
282
- yield events.BeforeExecution.from_operation(
283
- operation=operation,
284
- recursion_level=recursion_level,
285
- data_generation_method=data_generation_method,
286
- correlation_id=correlation_id,
287
- )
288
- hypothesis_output: List[str] = []
289
- errors: List[Exception] = []
290
- test_start_time = time.monotonic()
291
- setup_hypothesis_database_key(test, operation)
292
- try:
293
- with catch_warnings(record=True) as warnings, capture_hypothesis_output() as hypothesis_output:
294
- test(checks, targets, result, errors=errors, headers=headers, **kwargs)
295
- # Test body was not executed at all - Hypothesis did not generate any tests, but there is no error
296
- if not result.is_executed:
297
- status = Status.skip
298
- result.mark_skipped()
299
- else:
300
- status = Status.success
301
- except CheckFailed:
302
- status = Status.failure
303
- except NonCheckError:
304
- # It could be an error in user-defined extensions, network errors or internal Schemathesis errors
305
- status = Status.error
306
- result.mark_errored()
307
- for error in deduplicate_errors(errors):
308
- result.add_error(error)
309
- except hypothesis.errors.MultipleFailures:
310
- # Schemathesis may detect multiple errors that come from different check results
311
- # They raise different "grouped" exceptions, and `MultipleFailures` is risen as the result
312
- status = Status.failure
313
- except hypothesis.errors.Flaky:
314
- status = Status.failure
315
- result.mark_flaky()
316
- except hypothesis.errors.Unsatisfiable:
317
- # We need more clear error message here
318
- status = Status.error
319
- result.add_error(hypothesis.errors.Unsatisfiable("Unable to satisfy schema parameters for this API operation"))
320
- except KeyboardInterrupt:
321
- yield events.Interrupted()
322
- return
323
- except SkipTest:
324
- status = Status.skip
325
- result.mark_skipped()
326
- except AssertionError as exc: # comes from `hypothesis-jsonschema`
327
- error = reraise(exc)
328
- status = Status.error
329
- result.add_error(error)
330
- except HypothesisRefResolutionError:
331
- status = Status.error
332
- result.add_error(hypothesis.errors.Unsatisfiable(RECURSIVE_REFERENCE_ERROR_MESSAGE))
333
- except InvalidArgument as error:
334
- status = Status.error
335
- message = get_invalid_regular_expression_message(warnings)
336
- if message:
337
- # `hypothesis-jsonschema` emits a warning on invalid regular expression syntax
338
- result.add_error(InvalidRegularExpression(message))
339
- else:
340
- result.add_error(error)
341
- except hypothesis.errors.DeadlineExceeded as error:
342
- status = Status.error
343
- result.add_error(DeadlineExceeded.from_exc(error))
344
- except Exception as error:
345
- status = Status.error
346
- result.add_error(error)
347
- test_elapsed_time = time.monotonic() - test_start_time
348
- # Fetch seed value, hypothesis generates it during test execution
349
- # It may be `None` if the `derandomize` config option is set to `True`
350
- result.seed = getattr(test, "_hypothesis_internal_use_seed", None) or getattr(
351
- test, "_hypothesis_internal_use_generated_seed", None
352
- )
353
- results.append(result)
354
- yield events.AfterExecution.from_result(
355
- result=result,
356
- status=status,
357
- elapsed_time=test_elapsed_time,
358
- hypothesis_output=hypothesis_output,
359
- operation=operation,
360
- data_generation_method=data_generation_method,
361
- correlation_id=correlation_id,
362
- )
363
-
364
-
365
- def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
366
- """Make Hypothesis use separate database entries for every API operation.
367
-
368
- It increases the effectiveness of the Hypothesis database in the CLI.
369
- """
370
- # Hypothesis's function digest depends on the test function signature. To reflect it for the web API case,
371
- # we use all API operation parameters in the digest.
372
- extra = operation.verbose_name.encode("utf8")
373
- for parameter in operation.definition.parameters:
374
- extra += parameter.serialize(operation).encode("utf8")
375
- test.hypothesis.inner_test._hypothesis_internal_add_digest = extra # type: ignore
376
-
377
-
378
- def get_invalid_regular_expression_message(warnings: List[WarningMessage]) -> Optional[str]:
379
- for warning in warnings:
380
- message = str(warning.message)
381
- if "is not valid syntax for a Python regular expression" in message:
382
- return message
383
- return None
384
-
385
-
386
- def reraise(error: AssertionError) -> InvalidSchema:
387
- traceback = format_exception(error, True)
388
- if "assert type_ in TYPE_STRINGS" in traceback:
389
- message = "Invalid type name"
390
- else:
391
- message = "Unknown schema error"
392
- try:
393
- raise InvalidSchema(message) from error
394
- except InvalidSchema as exc:
395
- return exc
396
-
397
-
398
- def deduplicate_errors(errors: List[Exception]) -> Generator[Exception, None, None]:
399
- """Deduplicate errors by their messages + tracebacks."""
400
- seen = set()
401
- for error in errors:
402
- message = format_exception(error, True)
403
- if message in seen:
404
- continue
405
- seen.add(message)
406
- yield error
407
-
408
-
409
- def run_checks(
410
- case: Case,
411
- checks: Iterable[CheckFunction],
412
- check_results: List[Check],
413
- result: TestResult,
414
- response: GenericResponse,
415
- elapsed_time: float,
416
- max_response_time: Optional[int] = None,
417
- ) -> None:
418
- errors = []
419
-
420
- for check in checks:
421
- check_name = check.__name__
422
- copied_case = case.partial_deepcopy()
423
- copied_response = copy_response(response)
424
- try:
425
- skip_check = check(copied_response, copied_case)
426
- if not skip_check:
427
- check_result = result.add_success(check_name, copied_case, copied_response, elapsed_time)
428
- check_results.append(check_result)
429
- except AssertionError as exc:
430
- message = maybe_set_assertion_message(exc, check_name)
431
- errors.append(exc)
432
- if isinstance(exc, CheckFailed):
433
- context = exc.context
434
- else:
435
- context = None
436
- check_result = result.add_failure(check_name, copied_case, copied_response, elapsed_time, message, context)
437
- check_results.append(check_result)
438
-
439
- if max_response_time:
440
- if elapsed_time > max_response_time:
441
- message = f"Response time exceeded the limit of {max_response_time} ms"
442
- errors.append(AssertionError(message))
443
- result.add_failure(
444
- "max_response_time",
445
- case,
446
- response,
447
- elapsed_time,
448
- message,
449
- failures.ResponseTimeExceeded(elapsed=elapsed_time, deadline=max_response_time),
450
- )
451
- else:
452
- result.add_success("max_response_time", case, response, elapsed_time)
453
-
454
- if errors:
455
- raise get_grouped_exception(case.operation.verbose_name, *errors)(causes=tuple(errors))
456
-
457
-
458
- def run_targets(targets: Iterable[Callable], context: TargetContext) -> None:
459
- for target in targets:
460
- value = target(context)
461
- hypothesis.target(value, label=target.__name__)
462
-
463
-
464
- def add_cases(case: Case, response: GenericResponse, test: Callable, *args: Any) -> None:
465
- context = HookContext(case.operation)
466
- for case_hook in get_all_by_name("add_case"):
467
- _case = case_hook(context, case.partial_deepcopy(), response)
468
- # run additional test if _case is not an empty value
469
- if _case:
470
- test(_case, *args)
471
-
472
-
473
- @attr.s(slots=True) # pragma: no mutate
474
- class ErrorCollector:
475
- """Collect exceptions that are not related to failed checks.
476
-
477
- Such exceptions may be considered as multiple failures or flakiness by Hypothesis. In both cases, Hypothesis hides
478
- exception information that, in our case, is helpful for the end-user. It either indicates errors in user-defined
479
- extensions, network-related errors, or internal Schemathesis errors. In all cases, this information is useful for
480
- debugging.
481
-
482
- To mitigate this, we gather all exceptions manually via this context manager to avoid interfering with the test
483
- function signatures, which are used by Hypothesis.
484
- """
485
-
486
- errors: List[Exception] = attr.ib() # pragma: no mutate
487
-
488
- def __enter__(self) -> "ErrorCollector":
489
- return self
490
-
491
- # Typing: The return type suggested by mypy is `Literal[False]`, but I don't want to introduce dependency on the
492
- # `typing_extensions` package for Python 3.7
493
- def __exit__(
494
- self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
495
- ) -> Any:
496
- # Don't do anything special if:
497
- # - Tests are successful
498
- # - Checks failed
499
- # - The testing process is interrupted
500
- if not exc_type or issubclass(exc_type, CheckFailed) or not issubclass(exc_type, Exception):
501
- return False
502
- # These exceptions are needed for control flow on the Hypothesis side. E.g. rejecting unsatisfiable examples
503
- if isinstance(exc_val, HypothesisException):
504
- raise
505
- # Exception value is not `None` and is a subclass of `Exception` at this point
506
- exc_val = cast(Exception, exc_val)
507
- self.errors.append(exc_val.with_traceback(exc_tb))
508
- raise NonCheckError from None
509
-
510
-
511
- def network_test(
512
- case: Case,
513
- checks: Iterable[CheckFunction],
514
- targets: Iterable[Target],
515
- result: TestResult,
516
- session: requests.Session,
517
- request_timeout: Optional[int],
518
- request_tls_verify: bool,
519
- request_cert: Optional[RequestCert],
520
- store_interactions: bool,
521
- headers: Optional[Dict[str, Any]],
522
- feedback: Feedback,
523
- max_response_time: Optional[int],
524
- dry_run: bool,
525
- errors: List[Exception],
526
- ) -> None:
527
- """A single test body will be executed against the target."""
528
- with ErrorCollector(errors):
529
- result.mark_executed()
530
- headers = headers or {}
531
- if "user-agent" not in {header.lower() for header in headers}:
532
- headers["User-Agent"] = USER_AGENT
533
- timeout = prepare_timeout(request_timeout)
534
- if not dry_run:
535
- response = _network_test(
536
- case,
537
- checks,
538
- targets,
539
- result,
540
- session,
541
- timeout,
542
- store_interactions,
543
- headers,
544
- feedback,
545
- request_tls_verify,
546
- request_cert,
547
- max_response_time,
548
- )
549
- add_cases(
550
- case,
551
- response,
552
- _network_test,
553
- checks,
554
- targets,
555
- result,
556
- session,
557
- timeout,
558
- store_interactions,
559
- headers,
560
- feedback,
561
- request_tls_verify,
562
- request_cert,
563
- max_response_time,
564
- )
565
-
566
-
567
- def _network_test(
568
- case: Case,
569
- checks: Iterable[CheckFunction],
570
- targets: Iterable[Target],
571
- result: TestResult,
572
- session: requests.Session,
573
- timeout: Optional[float],
574
- store_interactions: bool,
575
- headers: Optional[Dict[str, Any]],
576
- feedback: Feedback,
577
- request_tls_verify: bool,
578
- request_cert: Optional[RequestCert],
579
- max_response_time: Optional[int],
580
- ) -> requests.Response:
581
- check_results: List[Check] = []
582
- try:
583
- hook_context = HookContext(operation=case.operation)
584
- hooks.dispatch("before_call", hook_context, case)
585
- kwargs: Dict[str, Any] = {
586
- "session": session,
587
- "headers": headers,
588
- "timeout": timeout,
589
- "verify": request_tls_verify,
590
- "cert": request_cert,
591
- }
592
- hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
593
- response = case.call(**kwargs)
594
- hooks.dispatch("after_call", hook_context, case, response)
595
- except CheckFailed as exc:
596
- check_name = "request_timeout"
597
- requests_kwargs = case.as_requests_kwargs(base_url=case.get_full_base_url(), headers=headers)
598
- request = requests.Request(**requests_kwargs).prepare()
599
- elapsed = cast(float, timeout) # It is defined and not empty, since the exception happened
600
- check_result = result.add_failure(
601
- check_name, case, None, elapsed, f"Response timed out after {1000 * elapsed:.2f}ms", exc.context, request
602
- )
603
- check_results.append(check_result)
604
- raise exc
605
- context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
606
- run_targets(targets, context)
607
- status = Status.success
608
- try:
609
- run_checks(case, checks, check_results, result, response, context.response_time * 1000, max_response_time)
610
- except CheckFailed:
611
- status = Status.failure
612
- raise
613
- finally:
614
- if store_interactions:
615
- result.store_requests_response(response, status, check_results)
616
- feedback.add_test_case(case, response)
617
- return response
618
-
619
-
620
- @contextmanager
621
- def get_session(auth: Optional[Union[HTTPDigestAuth, RawAuth]] = None) -> Generator[requests.Session, None, None]:
622
- with requests.Session() as session:
623
- if auth is not None:
624
- session.auth = auth
625
- yield session
626
-
627
-
628
- def prepare_timeout(timeout: Optional[int]) -> Optional[float]:
629
- """Request timeout is in milliseconds, but `requests` uses seconds."""
630
- output: Optional[Union[int, float]] = timeout
631
- if timeout is not None:
632
- output = timeout / 1000
633
- return output
634
-
635
-
636
- def wsgi_test(
637
- case: Case,
638
- checks: Iterable[CheckFunction],
639
- targets: Iterable[Target],
640
- result: TestResult,
641
- auth: Optional[RawAuth],
642
- auth_type: Optional[str],
643
- headers: Optional[Dict[str, Any]],
644
- store_interactions: bool,
645
- feedback: Feedback,
646
- max_response_time: Optional[int],
647
- dry_run: bool,
648
- errors: List[Exception],
649
- ) -> None:
650
- with ErrorCollector(errors):
651
- result.mark_executed()
652
- headers = _prepare_wsgi_headers(headers, auth, auth_type)
653
- if not dry_run:
654
- response = _wsgi_test(
655
- case, checks, targets, result, headers, store_interactions, feedback, max_response_time
656
- )
657
- add_cases(
658
- case,
659
- response,
660
- _wsgi_test,
661
- checks,
662
- targets,
663
- result,
664
- headers,
665
- store_interactions,
666
- feedback,
667
- max_response_time,
668
- )
669
-
670
-
671
- def _wsgi_test(
672
- case: Case,
673
- checks: Iterable[CheckFunction],
674
- targets: Iterable[Target],
675
- result: TestResult,
676
- headers: Dict[str, Any],
677
- store_interactions: bool,
678
- feedback: Feedback,
679
- max_response_time: Optional[int],
680
- ) -> WSGIResponse:
681
- with catching_logs(LogCaptureHandler(), level=logging.DEBUG) as recorded:
682
- start = time.monotonic()
683
- hook_context = HookContext(operation=case.operation)
684
- hooks.dispatch("before_call", hook_context, case)
685
- kwargs = {"headers": headers}
686
- hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
687
- response = case.call_wsgi(**kwargs)
688
- hooks.dispatch("after_call", hook_context, case, response)
689
- elapsed = time.monotonic() - start
690
- context = TargetContext(case=case, response=response, response_time=elapsed)
691
- run_targets(targets, context)
692
- result.logs.extend(recorded.records)
693
- status = Status.success
694
- check_results: List[Check] = []
695
- try:
696
- run_checks(case, checks, check_results, result, response, context.response_time * 1000, max_response_time)
697
- except CheckFailed:
698
- status = Status.failure
699
- raise
700
- finally:
701
- if store_interactions:
702
- result.store_wsgi_response(case, response, headers, elapsed, status, check_results)
703
- feedback.add_test_case(case, response)
704
- return response
705
-
706
-
707
- def _prepare_wsgi_headers(
708
- headers: Optional[Dict[str, Any]], auth: Optional[RawAuth], auth_type: Optional[str]
709
- ) -> Dict[str, Any]:
710
- headers = headers or {}
711
- if "user-agent" not in {header.lower() for header in headers}:
712
- headers["User-Agent"] = USER_AGENT
713
- wsgi_auth = get_wsgi_auth(auth, auth_type)
714
- if wsgi_auth:
715
- headers["Authorization"] = wsgi_auth
716
- return headers
717
-
718
-
719
- def get_wsgi_auth(auth: Optional[RawAuth], auth_type: Optional[str]) -> Optional[str]:
720
- if auth:
721
- if auth_type == "digest":
722
- raise ValueError("Digest auth is not supported for WSGI apps")
723
- return _basic_auth_str(*auth)
724
- return None
725
-
726
-
727
- def asgi_test(
728
- case: Case,
729
- checks: Iterable[CheckFunction],
730
- targets: Iterable[Target],
731
- result: TestResult,
732
- store_interactions: bool,
733
- headers: Optional[Dict[str, Any]],
734
- feedback: Feedback,
735
- max_response_time: Optional[int],
736
- dry_run: bool,
737
- errors: List[Exception],
738
- ) -> None:
739
- """A single test body will be executed against the target."""
740
- with ErrorCollector(errors):
741
- result.mark_executed()
742
- headers = headers or {}
743
-
744
- if not dry_run:
745
- response = _asgi_test(
746
- case, checks, targets, result, store_interactions, headers, feedback, max_response_time
747
- )
748
- add_cases(
749
- case,
750
- response,
751
- _asgi_test,
752
- checks,
753
- targets,
754
- result,
755
- store_interactions,
756
- headers,
757
- feedback,
758
- max_response_time,
759
- )
760
-
761
-
762
- def _asgi_test(
763
- case: Case,
764
- checks: Iterable[CheckFunction],
765
- targets: Iterable[Target],
766
- result: TestResult,
767
- store_interactions: bool,
768
- headers: Optional[Dict[str, Any]],
769
- feedback: Feedback,
770
- max_response_time: Optional[int],
771
- ) -> requests.Response:
772
- hook_context = HookContext(operation=case.operation)
773
- hooks.dispatch("before_call", hook_context, case)
774
- kwargs: Dict[str, Any] = {"headers": headers}
775
- hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
776
- response = case.call_asgi(**kwargs)
777
- hooks.dispatch("after_call", hook_context, case, response)
778
- context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
779
- run_targets(targets, context)
780
- status = Status.success
781
- check_results: List[Check] = []
782
- try:
783
- run_checks(case, checks, check_results, result, response, context.response_time * 1000, max_response_time)
784
- except CheckFailed:
785
- status = Status.failure
786
- raise
787
- finally:
788
- if store_interactions:
789
- result.store_requests_response(response, status, check_results)
790
- feedback.add_test_case(case, response)
791
- return response