schemathesis 3.15.4__py3-none-any.whl → 4.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. schemathesis/__init__.py +53 -25
  2. schemathesis/auths.py +507 -0
  3. schemathesis/checks.py +190 -25
  4. schemathesis/cli/__init__.py +27 -1219
  5. schemathesis/cli/__main__.py +4 -0
  6. schemathesis/cli/commands/__init__.py +133 -0
  7. schemathesis/cli/commands/data.py +10 -0
  8. schemathesis/cli/commands/run/__init__.py +602 -0
  9. schemathesis/cli/commands/run/context.py +228 -0
  10. schemathesis/cli/commands/run/events.py +60 -0
  11. schemathesis/cli/commands/run/executor.py +157 -0
  12. schemathesis/cli/commands/run/filters.py +53 -0
  13. schemathesis/cli/commands/run/handlers/__init__.py +46 -0
  14. schemathesis/cli/commands/run/handlers/base.py +45 -0
  15. schemathesis/cli/commands/run/handlers/cassettes.py +464 -0
  16. schemathesis/cli/commands/run/handlers/junitxml.py +60 -0
  17. schemathesis/cli/commands/run/handlers/output.py +1750 -0
  18. schemathesis/cli/commands/run/loaders.py +118 -0
  19. schemathesis/cli/commands/run/validation.py +256 -0
  20. schemathesis/cli/constants.py +5 -0
  21. schemathesis/cli/core.py +19 -0
  22. schemathesis/cli/ext/fs.py +16 -0
  23. schemathesis/cli/ext/groups.py +203 -0
  24. schemathesis/cli/ext/options.py +81 -0
  25. schemathesis/config/__init__.py +202 -0
  26. schemathesis/config/_auth.py +51 -0
  27. schemathesis/config/_checks.py +268 -0
  28. schemathesis/config/_diff_base.py +101 -0
  29. schemathesis/config/_env.py +21 -0
  30. schemathesis/config/_error.py +163 -0
  31. schemathesis/config/_generation.py +157 -0
  32. schemathesis/config/_health_check.py +24 -0
  33. schemathesis/config/_operations.py +335 -0
  34. schemathesis/config/_output.py +171 -0
  35. schemathesis/config/_parameters.py +19 -0
  36. schemathesis/config/_phases.py +253 -0
  37. schemathesis/config/_projects.py +543 -0
  38. schemathesis/config/_rate_limit.py +17 -0
  39. schemathesis/config/_report.py +120 -0
  40. schemathesis/config/_validator.py +9 -0
  41. schemathesis/config/_warnings.py +89 -0
  42. schemathesis/config/schema.json +975 -0
  43. schemathesis/core/__init__.py +72 -0
  44. schemathesis/core/adapter.py +34 -0
  45. schemathesis/core/compat.py +32 -0
  46. schemathesis/core/control.py +2 -0
  47. schemathesis/core/curl.py +100 -0
  48. schemathesis/core/deserialization.py +210 -0
  49. schemathesis/core/errors.py +588 -0
  50. schemathesis/core/failures.py +316 -0
  51. schemathesis/core/fs.py +19 -0
  52. schemathesis/core/hooks.py +20 -0
  53. schemathesis/core/jsonschema/__init__.py +13 -0
  54. schemathesis/core/jsonschema/bundler.py +183 -0
  55. schemathesis/core/jsonschema/keywords.py +40 -0
  56. schemathesis/core/jsonschema/references.py +222 -0
  57. schemathesis/core/jsonschema/types.py +41 -0
  58. schemathesis/core/lazy_import.py +15 -0
  59. schemathesis/core/loaders.py +107 -0
  60. schemathesis/core/marks.py +66 -0
  61. schemathesis/core/media_types.py +79 -0
  62. schemathesis/core/output/__init__.py +46 -0
  63. schemathesis/core/output/sanitization.py +54 -0
  64. schemathesis/core/parameters.py +45 -0
  65. schemathesis/core/rate_limit.py +60 -0
  66. schemathesis/core/registries.py +34 -0
  67. schemathesis/core/result.py +27 -0
  68. schemathesis/core/schema_analysis.py +17 -0
  69. schemathesis/core/shell.py +203 -0
  70. schemathesis/core/transforms.py +144 -0
  71. schemathesis/core/transport.py +223 -0
  72. schemathesis/core/validation.py +73 -0
  73. schemathesis/core/version.py +7 -0
  74. schemathesis/engine/__init__.py +28 -0
  75. schemathesis/engine/context.py +152 -0
  76. schemathesis/engine/control.py +44 -0
  77. schemathesis/engine/core.py +201 -0
  78. schemathesis/engine/errors.py +446 -0
  79. schemathesis/engine/events.py +284 -0
  80. schemathesis/engine/observations.py +42 -0
  81. schemathesis/engine/phases/__init__.py +108 -0
  82. schemathesis/engine/phases/analysis.py +28 -0
  83. schemathesis/engine/phases/probes.py +172 -0
  84. schemathesis/engine/phases/stateful/__init__.py +68 -0
  85. schemathesis/engine/phases/stateful/_executor.py +364 -0
  86. schemathesis/engine/phases/stateful/context.py +85 -0
  87. schemathesis/engine/phases/unit/__init__.py +220 -0
  88. schemathesis/engine/phases/unit/_executor.py +459 -0
  89. schemathesis/engine/phases/unit/_pool.py +82 -0
  90. schemathesis/engine/recorder.py +254 -0
  91. schemathesis/errors.py +47 -0
  92. schemathesis/filters.py +395 -0
  93. schemathesis/generation/__init__.py +25 -0
  94. schemathesis/generation/case.py +478 -0
  95. schemathesis/generation/coverage.py +1528 -0
  96. schemathesis/generation/hypothesis/__init__.py +121 -0
  97. schemathesis/generation/hypothesis/builder.py +992 -0
  98. schemathesis/generation/hypothesis/examples.py +56 -0
  99. schemathesis/generation/hypothesis/given.py +66 -0
  100. schemathesis/generation/hypothesis/reporting.py +285 -0
  101. schemathesis/generation/meta.py +227 -0
  102. schemathesis/generation/metrics.py +93 -0
  103. schemathesis/generation/modes.py +20 -0
  104. schemathesis/generation/overrides.py +127 -0
  105. schemathesis/generation/stateful/__init__.py +37 -0
  106. schemathesis/generation/stateful/state_machine.py +294 -0
  107. schemathesis/graphql/__init__.py +15 -0
  108. schemathesis/graphql/checks.py +109 -0
  109. schemathesis/graphql/loaders.py +285 -0
  110. schemathesis/hooks.py +270 -91
  111. schemathesis/openapi/__init__.py +13 -0
  112. schemathesis/openapi/checks.py +467 -0
  113. schemathesis/openapi/generation/__init__.py +0 -0
  114. schemathesis/openapi/generation/filters.py +72 -0
  115. schemathesis/openapi/loaders.py +315 -0
  116. schemathesis/pytest/__init__.py +5 -0
  117. schemathesis/pytest/control_flow.py +7 -0
  118. schemathesis/pytest/lazy.py +341 -0
  119. schemathesis/pytest/loaders.py +36 -0
  120. schemathesis/pytest/plugin.py +357 -0
  121. schemathesis/python/__init__.py +0 -0
  122. schemathesis/python/asgi.py +12 -0
  123. schemathesis/python/wsgi.py +12 -0
  124. schemathesis/schemas.py +682 -257
  125. schemathesis/specs/graphql/__init__.py +0 -1
  126. schemathesis/specs/graphql/nodes.py +26 -2
  127. schemathesis/specs/graphql/scalars.py +77 -12
  128. schemathesis/specs/graphql/schemas.py +367 -148
  129. schemathesis/specs/graphql/validation.py +33 -0
  130. schemathesis/specs/openapi/__init__.py +9 -1
  131. schemathesis/specs/openapi/_hypothesis.py +555 -318
  132. schemathesis/specs/openapi/adapter/__init__.py +10 -0
  133. schemathesis/specs/openapi/adapter/parameters.py +729 -0
  134. schemathesis/specs/openapi/adapter/protocol.py +59 -0
  135. schemathesis/specs/openapi/adapter/references.py +19 -0
  136. schemathesis/specs/openapi/adapter/responses.py +368 -0
  137. schemathesis/specs/openapi/adapter/security.py +144 -0
  138. schemathesis/specs/openapi/adapter/v2.py +30 -0
  139. schemathesis/specs/openapi/adapter/v3_0.py +30 -0
  140. schemathesis/specs/openapi/adapter/v3_1.py +30 -0
  141. schemathesis/specs/openapi/analysis.py +96 -0
  142. schemathesis/specs/openapi/checks.py +748 -82
  143. schemathesis/specs/openapi/converter.py +176 -37
  144. schemathesis/specs/openapi/definitions.py +599 -4
  145. schemathesis/specs/openapi/examples.py +581 -165
  146. schemathesis/specs/openapi/expressions/__init__.py +52 -5
  147. schemathesis/specs/openapi/expressions/extractors.py +25 -0
  148. schemathesis/specs/openapi/expressions/lexer.py +34 -31
  149. schemathesis/specs/openapi/expressions/nodes.py +97 -46
  150. schemathesis/specs/openapi/expressions/parser.py +35 -13
  151. schemathesis/specs/openapi/formats.py +122 -0
  152. schemathesis/specs/openapi/media_types.py +75 -0
  153. schemathesis/specs/openapi/negative/__init__.py +93 -73
  154. schemathesis/specs/openapi/negative/mutations.py +294 -103
  155. schemathesis/specs/openapi/negative/utils.py +0 -9
  156. schemathesis/specs/openapi/patterns.py +458 -0
  157. schemathesis/specs/openapi/references.py +60 -81
  158. schemathesis/specs/openapi/schemas.py +647 -666
  159. schemathesis/specs/openapi/serialization.py +53 -30
  160. schemathesis/specs/openapi/stateful/__init__.py +403 -68
  161. schemathesis/specs/openapi/stateful/control.py +87 -0
  162. schemathesis/specs/openapi/stateful/dependencies/__init__.py +232 -0
  163. schemathesis/specs/openapi/stateful/dependencies/inputs.py +428 -0
  164. schemathesis/specs/openapi/stateful/dependencies/models.py +341 -0
  165. schemathesis/specs/openapi/stateful/dependencies/naming.py +491 -0
  166. schemathesis/specs/openapi/stateful/dependencies/outputs.py +34 -0
  167. schemathesis/specs/openapi/stateful/dependencies/resources.py +339 -0
  168. schemathesis/specs/openapi/stateful/dependencies/schemas.py +447 -0
  169. schemathesis/specs/openapi/stateful/inference.py +254 -0
  170. schemathesis/specs/openapi/stateful/links.py +219 -78
  171. schemathesis/specs/openapi/types/__init__.py +3 -0
  172. schemathesis/specs/openapi/types/common.py +23 -0
  173. schemathesis/specs/openapi/types/v2.py +129 -0
  174. schemathesis/specs/openapi/types/v3.py +134 -0
  175. schemathesis/specs/openapi/utils.py +7 -6
  176. schemathesis/specs/openapi/warnings.py +75 -0
  177. schemathesis/transport/__init__.py +224 -0
  178. schemathesis/transport/asgi.py +26 -0
  179. schemathesis/transport/prepare.py +126 -0
  180. schemathesis/transport/requests.py +278 -0
  181. schemathesis/transport/serialization.py +329 -0
  182. schemathesis/transport/wsgi.py +175 -0
  183. schemathesis-4.4.2.dist-info/METADATA +213 -0
  184. schemathesis-4.4.2.dist-info/RECORD +192 -0
  185. {schemathesis-3.15.4.dist-info → schemathesis-4.4.2.dist-info}/WHEEL +1 -1
  186. schemathesis-4.4.2.dist-info/entry_points.txt +6 -0
  187. {schemathesis-3.15.4.dist-info → schemathesis-4.4.2.dist-info/licenses}/LICENSE +1 -1
  188. schemathesis/_compat.py +0 -57
  189. schemathesis/_hypothesis.py +0 -123
  190. schemathesis/auth.py +0 -214
  191. schemathesis/cli/callbacks.py +0 -240
  192. schemathesis/cli/cassettes.py +0 -351
  193. schemathesis/cli/context.py +0 -38
  194. schemathesis/cli/debug.py +0 -21
  195. schemathesis/cli/handlers.py +0 -11
  196. schemathesis/cli/junitxml.py +0 -41
  197. schemathesis/cli/options.py +0 -70
  198. schemathesis/cli/output/__init__.py +0 -1
  199. schemathesis/cli/output/default.py +0 -521
  200. schemathesis/cli/output/short.py +0 -40
  201. schemathesis/constants.py +0 -88
  202. schemathesis/exceptions.py +0 -257
  203. schemathesis/extra/_aiohttp.py +0 -27
  204. schemathesis/extra/_flask.py +0 -10
  205. schemathesis/extra/_server.py +0 -16
  206. schemathesis/extra/pytest_plugin.py +0 -251
  207. schemathesis/failures.py +0 -145
  208. schemathesis/fixups/__init__.py +0 -29
  209. schemathesis/fixups/fast_api.py +0 -30
  210. schemathesis/graphql.py +0 -5
  211. schemathesis/internal.py +0 -6
  212. schemathesis/lazy.py +0 -301
  213. schemathesis/models.py +0 -1113
  214. schemathesis/parameters.py +0 -91
  215. schemathesis/runner/__init__.py +0 -470
  216. schemathesis/runner/events.py +0 -242
  217. schemathesis/runner/impl/__init__.py +0 -3
  218. schemathesis/runner/impl/core.py +0 -791
  219. schemathesis/runner/impl/solo.py +0 -85
  220. schemathesis/runner/impl/threadpool.py +0 -367
  221. schemathesis/runner/serialization.py +0 -206
  222. schemathesis/serializers.py +0 -253
  223. schemathesis/service/__init__.py +0 -18
  224. schemathesis/service/auth.py +0 -10
  225. schemathesis/service/client.py +0 -62
  226. schemathesis/service/constants.py +0 -25
  227. schemathesis/service/events.py +0 -39
  228. schemathesis/service/handler.py +0 -46
  229. schemathesis/service/hosts.py +0 -74
  230. schemathesis/service/metadata.py +0 -42
  231. schemathesis/service/models.py +0 -21
  232. schemathesis/service/serialization.py +0 -184
  233. schemathesis/service/worker.py +0 -39
  234. schemathesis/specs/graphql/loaders.py +0 -215
  235. schemathesis/specs/openapi/constants.py +0 -7
  236. schemathesis/specs/openapi/expressions/context.py +0 -12
  237. schemathesis/specs/openapi/expressions/pointers.py +0 -29
  238. schemathesis/specs/openapi/filters.py +0 -44
  239. schemathesis/specs/openapi/links.py +0 -303
  240. schemathesis/specs/openapi/loaders.py +0 -453
  241. schemathesis/specs/openapi/parameters.py +0 -430
  242. schemathesis/specs/openapi/security.py +0 -129
  243. schemathesis/specs/openapi/validation.py +0 -24
  244. schemathesis/stateful.py +0 -358
  245. schemathesis/targets.py +0 -32
  246. schemathesis/types.py +0 -38
  247. schemathesis/utils.py +0 -475
  248. schemathesis-3.15.4.dist-info/METADATA +0 -202
  249. schemathesis-3.15.4.dist-info/RECORD +0 -99
  250. schemathesis-3.15.4.dist-info/entry_points.txt +0 -7
  251. /schemathesis/{extra → cli/ext}/__init__.py +0 -0
@@ -0,0 +1,1750 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import textwrap
5
+ import time
6
+ from dataclasses import dataclass, field
7
+ from itertools import groupby
8
+ from json.decoder import JSONDecodeError
9
+ from types import GeneratorType
10
+ from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable
11
+
12
+ import click
13
+
14
+ from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
15
+ from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
16
+ from schemathesis.cli.commands.run.handlers.base import EventHandler
17
+ from schemathesis.cli.constants import ISSUE_TRACKER_URL
18
+ from schemathesis.cli.core import get_terminal_width
19
+ from schemathesis.config import ProjectConfig, ReportFormat, SchemathesisWarning
20
+ from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
21
+ from schemathesis.core.failures import MessageBlock, Severity, format_failures
22
+ from schemathesis.core.output import prepare_response_payload
23
+ from schemathesis.core.parameters import ParameterLocation
24
+ from schemathesis.core.result import Ok
25
+ from schemathesis.core.version import SCHEMATHESIS_VERSION
26
+ from schemathesis.engine import Status, events
27
+ from schemathesis.engine.phases import PhaseName, PhaseSkipReason
28
+ from schemathesis.engine.phases.probes import ProbeOutcome
29
+ from schemathesis.engine.recorder import Interaction, ScenarioRecorder
30
+ from schemathesis.generation.meta import CoveragePhaseData, CoverageScenario
31
+ from schemathesis.generation.modes import GenerationMode
32
+ from schemathesis.schemas import ApiStatistic
33
+
34
+ if TYPE_CHECKING:
35
+ from rich.console import Console, Group
36
+ from rich.live import Live
37
+ from rich.progress import Progress, TaskID
38
+ from rich.text import Text
39
+
40
+ from schemathesis.generation.stateful.state_machine import ExtractionFailure
41
+
42
+ IO_ENCODING = os.getenv("PYTHONIOENCODING", "utf-8")
43
+ DISCORD_LINK = "https://discord.gg/R9ASRAmHnA"
44
+
45
+
46
+ def display_section_name(title: str, separator: str = "=", **kwargs: Any) -> None:
47
+ """Print section name with separators in terminal with the given title nicely centered."""
48
+ message = f" {title} ".center(get_terminal_width(), separator)
49
+ kwargs.setdefault("bold", True)
50
+ click.echo(_style(message, **kwargs))
51
+
52
+
53
+ def bold(option: str) -> str:
54
+ return click.style(option, bold=True)
55
+
56
+
57
+ def display_failures(ctx: ExecutionContext) -> None:
58
+ """Display all failures in the test run."""
59
+ if not ctx.statistic.failures:
60
+ return
61
+
62
+ display_section_name("FAILURES")
63
+ for label, failures in ctx.statistic.failures.items():
64
+ display_failures_for_single_test(ctx, label, failures.values())
65
+
66
+
67
+ if IO_ENCODING != "utf-8":
68
+ HEADER_SEPARATOR = "-"
69
+
70
+ def _style(text: str, **kwargs: Any) -> str:
71
+ text = text.encode(IO_ENCODING, errors="replace").decode("utf-8")
72
+ return click.style(text, **kwargs)
73
+
74
+ else:
75
+ HEADER_SEPARATOR = "━"
76
+
77
+ def _style(text: str, **kwargs: Any) -> str:
78
+ return click.style(text, **kwargs)
79
+
80
+
81
+ def failure_formatter(block: MessageBlock, content: str) -> str:
82
+ if block == MessageBlock.CASE_ID:
83
+ return _style(content, bold=True)
84
+ if block == MessageBlock.FAILURE:
85
+ return _style(content, fg="red", bold=True)
86
+ if block == MessageBlock.STATUS:
87
+ return _style(content, bold=True)
88
+ assert block == MessageBlock.CURL
89
+ return _style(content.replace("Reproduce with", bold("Reproduce with")))
90
+
91
+
92
+ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks: Iterable[GroupedFailures]) -> None:
93
+ """Display a failure for a single method / path."""
94
+ display_section_name(label, "_", fg="red")
95
+ for idx, group in enumerate(checks, 1):
96
+ click.echo(
97
+ format_failures(
98
+ case_id=f"{idx}. Test Case ID: {group.case_id}",
99
+ response=group.response,
100
+ failures=group.failures,
101
+ curl=group.code_sample,
102
+ formatter=failure_formatter,
103
+ config=ctx.config.output,
104
+ )
105
+ )
106
+ click.echo()
107
+
108
+
109
+ VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema or GraphQL endpoint"
110
+ DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--tls-verify=false`')}."
111
+ LOADER_ERROR_SUGGESTIONS = {
112
+ # SSL-specific connection issue
113
+ LoaderErrorKind.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
114
+ # Other connection problems
115
+ LoaderErrorKind.CONNECTION_OTHER: f"Use {bold('`--wait-for-schema=NUM`')} to wait up to NUM seconds for schema availability.",
116
+ # Response issues
117
+ LoaderErrorKind.UNEXPECTED_CONTENT_TYPE: VERIFY_URL_SUGGESTION,
118
+ LoaderErrorKind.HTTP_FORBIDDEN: "Verify your API keys or authentication headers.",
119
+ LoaderErrorKind.HTTP_NOT_FOUND: VERIFY_URL_SUGGESTION,
120
+ # OpenAPI specification issues
121
+ LoaderErrorKind.OPEN_API_UNSPECIFIED_VERSION: "Include the version in the schema.",
122
+ # YAML specific issues
123
+ LoaderErrorKind.YAML_NUMERIC_STATUS_CODES: "Convert numeric status codes to strings.",
124
+ LoaderErrorKind.YAML_NON_STRING_KEYS: "Convert non-string keys to strings.",
125
+ # Unclassified
126
+ LoaderErrorKind.UNCLASSIFIED: f"If you suspect this is a Schemathesis issue and the schema is valid, please report it and include the schema if you can:\n\n {ISSUE_TRACKER_URL}",
127
+ }
128
+
129
+
130
+ def _display_extras(extras: list[str]) -> None:
131
+ if extras:
132
+ click.echo()
133
+ for extra in extras:
134
+ click.echo(_style(f" {extra}"))
135
+
136
+
137
+ def display_header(version: str) -> None:
138
+ prefix = "v" if version != "dev" else ""
139
+ header = f"Schemathesis {prefix}{version}"
140
+ click.echo(_style(header, bold=True))
141
+ click.echo(_style(HEADER_SEPARATOR * len(header), bold=True))
142
+ click.echo()
143
+
144
+
145
+ DEFAULT_INTERNAL_ERROR_MESSAGE = "An internal error occurred during the test run"
146
+ TRUNCATION_PLACEHOLDER = "[...]"
147
+
148
+
149
+ def _print_lines(lines: list[str | Generator[str, None, None]]) -> None:
150
+ for entry in lines:
151
+ if isinstance(entry, str):
152
+ click.echo(entry)
153
+ elif isinstance(entry, GeneratorType):
154
+ for line in entry:
155
+ click.echo(line)
156
+
157
+
158
+ def _default_console() -> Console:
159
+ from rich.console import Console
160
+
161
+ kwargs = {}
162
+ # For stdout recording in tests
163
+ if "PYTEST_VERSION" in os.environ:
164
+ kwargs["width"] = 240
165
+ return Console(**kwargs)
166
+
167
+
168
+ BLOCK_PADDING = (0, 1, 0, 1)
169
+
170
+
171
+ @dataclass
172
+ class LoadingProgressManager:
173
+ console: Console
174
+ location: str
175
+ start_time: float
176
+ progress: Progress
177
+ progress_task_id: TaskID | None
178
+ is_interrupted: bool
179
+
180
+ __slots__ = ("console", "location", "start_time", "progress", "progress_task_id", "is_interrupted")
181
+
182
+ def __init__(self, console: Console, location: str) -> None:
183
+ from rich.progress import Progress, RenderableColumn, SpinnerColumn, TextColumn
184
+ from rich.style import Style
185
+ from rich.text import Text
186
+
187
+ self.console = console
188
+ self.location = location
189
+ self.start_time = time.monotonic()
190
+ progress_message = Text.assemble(
191
+ ("Loading specification from ", Style(color="white")),
192
+ (location, Style(color="cyan")),
193
+ )
194
+ self.progress = Progress(
195
+ TextColumn(""),
196
+ SpinnerColumn("clock"),
197
+ RenderableColumn(progress_message),
198
+ console=console,
199
+ transient=True,
200
+ )
201
+ self.progress_task_id = None
202
+ self.is_interrupted = False
203
+
204
+ def start(self) -> None:
205
+ """Start loading progress display."""
206
+ self.progress_task_id = self.progress.add_task("Loading", total=None)
207
+ self.progress.start()
208
+
209
+ def stop(self) -> None:
210
+ """Stop loading progress display."""
211
+ assert self.progress_task_id is not None
212
+ self.progress.stop_task(self.progress_task_id)
213
+ self.progress.stop()
214
+
215
+ def interrupt(self) -> None:
216
+ """Handle interruption during loading."""
217
+ self.is_interrupted = True
218
+ self.stop()
219
+
220
+ def get_completion_message(self) -> Text:
221
+ """Generate completion message including duration."""
222
+ from rich.style import Style
223
+ from rich.text import Text
224
+
225
+ duration = format_duration(int((time.monotonic() - self.start_time) * 1000))
226
+ if self.is_interrupted:
227
+ return Text.assemble(
228
+ ("⚡ ", Style(color="yellow")),
229
+ (f"Loading interrupted after {duration} while loading from ", Style(color="white")),
230
+ (self.location, Style(color="cyan")),
231
+ )
232
+ return Text.assemble(
233
+ ("✅ ", Style(color="green")),
234
+ ("Loaded specification from ", Style(color="bright_white")),
235
+ (self.location, Style(color="cyan")),
236
+ (f" (in {duration})", Style(color="bright_white")),
237
+ )
238
+
239
+ def get_error_message(self, error: LoaderError) -> Group:
240
+ from rich.console import Group
241
+ from rich.style import Style
242
+ from rich.text import Text
243
+
244
+ duration = format_duration(int((time.monotonic() - self.start_time) * 1000))
245
+
246
+ # Show what was attempted
247
+ attempted = Text.assemble(
248
+ ("❌ ", Style(color="red")),
249
+ ("Failed to load specification from ", Style(color="white")),
250
+ (self.location, Style(color="cyan")),
251
+ (f" after {duration}", Style(color="white")),
252
+ )
253
+
254
+ # Show error details
255
+ error_title = Text("Schema Loading Error", style=Style(color="red", bold=True))
256
+ error_message = Text(error.message)
257
+
258
+ return Group(
259
+ attempted,
260
+ Text(),
261
+ error_title,
262
+ Text(),
263
+ error_message,
264
+ )
265
+
266
+
267
+ @dataclass
268
+ class ProbingProgressManager:
269
+ console: Console
270
+ start_time: float
271
+ progress: Progress
272
+ progress_task_id: TaskID | None
273
+ is_interrupted: bool
274
+
275
+ __slots__ = ("console", "start_time", "progress", "progress_task_id", "is_interrupted")
276
+
277
+ def __init__(self, console: Console) -> None:
278
+ from rich.progress import Progress, RenderableColumn, SpinnerColumn, TextColumn
279
+ from rich.text import Text
280
+
281
+ self.console = console
282
+ self.start_time = time.monotonic()
283
+ self.progress = Progress(
284
+ TextColumn(""),
285
+ SpinnerColumn("clock"),
286
+ RenderableColumn(Text("Probing API capabilities", style="bright_white")),
287
+ transient=True,
288
+ console=console,
289
+ )
290
+ self.progress_task_id = None
291
+ self.is_interrupted = False
292
+
293
+ def start(self) -> None:
294
+ """Start probing progress display."""
295
+ self.progress_task_id = self.progress.add_task("Probing", total=None)
296
+ self.progress.start()
297
+
298
+ def stop(self) -> None:
299
+ """Stop probing progress display."""
300
+ assert self.progress_task_id is not None
301
+ self.progress.stop_task(self.progress_task_id)
302
+ self.progress.stop()
303
+
304
+ def interrupt(self) -> None:
305
+ """Handle interruption during probing."""
306
+ self.is_interrupted = True
307
+ self.stop()
308
+
309
+ def get_completion_message(self) -> Text:
310
+ """Generate completion message including duration."""
311
+ from rich.style import Style
312
+ from rich.text import Text
313
+
314
+ duration = format_duration(int((time.monotonic() - self.start_time) * 1000))
315
+ if self.is_interrupted:
316
+ return Text.assemble(
317
+ ("⚡ ", Style(color="yellow")),
318
+ (f"API probing interrupted after {duration}", Style(color="white")),
319
+ )
320
+ return Text.assemble(
321
+ ("✅ ", Style(color="green")),
322
+ ("API capabilities:", Style(color="white")),
323
+ )
324
+
325
+
326
+ @dataclass
327
+ class WarningData:
328
+ missing_auth: dict[int, set[str]]
329
+ missing_test_data: set[str]
330
+ validation_mismatch: set[str]
331
+ missing_deserializer: dict[str, set[str]]
332
+
333
+ __slots__ = ("missing_auth", "missing_test_data", "validation_mismatch", "missing_deserializer")
334
+
335
+ def __init__(
336
+ self,
337
+ missing_auth: dict[int, set[str]] | None = None,
338
+ missing_test_data: set[str] | None = None,
339
+ validation_mismatch: set[str] | None = None,
340
+ missing_deserializer: dict[str, set[str]] | None = None,
341
+ ) -> None:
342
+ self.missing_auth = missing_auth or {}
343
+ self.missing_test_data = missing_test_data or set()
344
+ self.validation_mismatch = validation_mismatch or set()
345
+ self.missing_deserializer = missing_deserializer or {}
346
+
347
+ @property
348
+ def is_empty(self) -> bool:
349
+ return not bool(
350
+ self.missing_auth or self.missing_test_data or self.validation_mismatch or self.missing_deserializer
351
+ )
352
+
353
+
354
+ @dataclass
355
+ class OperationProgress:
356
+ """Tracks individual operation progress."""
357
+
358
+ label: str
359
+ start_time: float
360
+ task_id: TaskID
361
+
362
+ __slots__ = ("label", "start_time", "task_id")
363
+
364
+
365
+ @dataclass
366
+ class UnitTestProgressManager:
367
+ """Manages progress display for unit tests."""
368
+
369
+ console: Console
370
+ title: str
371
+ current: int
372
+ total: int
373
+ start_time: float
374
+
375
+ # Progress components
376
+ title_progress: Progress
377
+ progress_bar: Progress
378
+ operations_progress: Progress
379
+ current_operations: dict[str, OperationProgress]
380
+ stats: dict[Status, int]
381
+ stats_progress: Progress
382
+ live: Live | None
383
+
384
+ # Task IDs
385
+ title_task_id: TaskID | None
386
+ progress_task_id: TaskID | None
387
+ stats_task_id: TaskID
388
+
389
+ is_interrupted: bool
390
+
391
+ __slots__ = (
392
+ "console",
393
+ "title",
394
+ "current",
395
+ "total",
396
+ "start_time",
397
+ "title_progress",
398
+ "progress_bar",
399
+ "operations_progress",
400
+ "current_operations",
401
+ "stats",
402
+ "stats_progress",
403
+ "live",
404
+ "title_task_id",
405
+ "progress_task_id",
406
+ "stats_task_id",
407
+ "is_interrupted",
408
+ )
409
+
410
+ def __init__(
411
+ self,
412
+ *,
413
+ console: Console,
414
+ title: str,
415
+ total: int,
416
+ ) -> None:
417
+ from rich.progress import (
418
+ BarColumn,
419
+ Progress,
420
+ SpinnerColumn,
421
+ TextColumn,
422
+ TimeElapsedColumn,
423
+ )
424
+ from rich.style import Style
425
+
426
+ self.console = console
427
+ self.title = title
428
+ self.current = 0
429
+ self.total = total
430
+ self.start_time = time.monotonic()
431
+
432
+ # Initialize progress displays
433
+ self.title_progress = Progress(
434
+ TextColumn(""),
435
+ SpinnerColumn("clock"),
436
+ TextColumn("{task.description}", style=Style(color="white")),
437
+ console=self.console,
438
+ )
439
+ self.title_task_id = None
440
+
441
+ self.progress_bar = Progress(
442
+ TextColumn(" "),
443
+ TimeElapsedColumn(),
444
+ BarColumn(bar_width=None),
445
+ TextColumn("{task.percentage:.0f}% ({task.completed}/{task.total})"),
446
+ console=self.console,
447
+ )
448
+ self.progress_task_id = None
449
+
450
+ self.operations_progress = Progress(
451
+ TextColumn(" "),
452
+ SpinnerColumn("dots"),
453
+ TimeElapsedColumn(),
454
+ TextColumn(" {task.fields[label]}"),
455
+ console=self.console,
456
+ )
457
+
458
+ self.current_operations = {}
459
+
460
+ self.stats_progress = Progress(
461
+ TextColumn(" "),
462
+ TextColumn("{task.description}"),
463
+ console=self.console,
464
+ )
465
+ self.stats_task_id = self.stats_progress.add_task("")
466
+ self.stats = {
467
+ Status.SUCCESS: 0,
468
+ Status.FAILURE: 0,
469
+ Status.SKIP: 0,
470
+ Status.ERROR: 0,
471
+ Status.INTERRUPTED: 0,
472
+ }
473
+ self._update_stats_display()
474
+
475
+ self.live = None
476
+ self.is_interrupted = False
477
+
478
+ def _get_stats_message(self) -> str:
479
+ width = len(str(self.total))
480
+
481
+ parts = []
482
+ if self.stats[Status.SUCCESS]:
483
+ parts.append(f"✅ {self.stats[Status.SUCCESS]:{width}d} passed")
484
+ if self.stats[Status.FAILURE]:
485
+ parts.append(f"❌ {self.stats[Status.FAILURE]:{width}d} failed")
486
+ if self.stats[Status.ERROR]:
487
+ suffix = "s" if self.stats[Status.ERROR] > 1 else ""
488
+ parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} error{suffix}")
489
+ if self.stats[Status.SKIP] or self.stats[Status.INTERRUPTED]:
490
+ parts.append(f"⏭ {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
491
+ return " ".join(parts)
492
+
493
+ def _update_stats_display(self) -> None:
494
+ """Update the statistics display."""
495
+ self.stats_progress.update(self.stats_task_id, description=self._get_stats_message())
496
+
497
+ def start(self) -> None:
498
+ """Start progress display."""
499
+ from rich.console import Group
500
+ from rich.live import Live
501
+ from rich.text import Text
502
+
503
+ group = Group(
504
+ self.title_progress,
505
+ Text(),
506
+ self.progress_bar,
507
+ Text(),
508
+ self.operations_progress,
509
+ Text(),
510
+ self.stats_progress,
511
+ )
512
+
513
+ self.live = Live(group, refresh_per_second=10, console=self.console, transient=True)
514
+ self.live.start()
515
+
516
+ # Initialize both progress displays
517
+ self.title_task_id = self.title_progress.add_task(self.title, total=self.total)
518
+ self.progress_task_id = self.progress_bar.add_task(
519
+ "", # Empty description as it's shown in title
520
+ total=self.total,
521
+ )
522
+
523
+ def update_progress(self) -> None:
524
+ """Update progress in both displays."""
525
+ assert self.title_task_id is not None
526
+ assert self.progress_task_id is not None
527
+
528
+ self.current += 1
529
+ self.title_progress.update(self.title_task_id, completed=self.current)
530
+ self.progress_bar.update(self.progress_task_id, completed=self.current)
531
+
532
+ def start_operation(self, label: str) -> None:
533
+ """Start tracking new operation."""
534
+ task_id = self.operations_progress.add_task("", label=label, start_time=time.monotonic())
535
+ self.current_operations[label] = OperationProgress(label=label, start_time=time.monotonic(), task_id=task_id)
536
+
537
+ def finish_operation(self, label: str) -> None:
538
+ """Finish tracking operation."""
539
+ if operation := self.current_operations.pop(label, None):
540
+ if not self.current_operations:
541
+ assert self.title_task_id is not None
542
+ if self.current == self.total - 1:
543
+ description = f" {self.title}"
544
+ else:
545
+ description = self.title
546
+ self.title_progress.update(self.title_task_id, description=description)
547
+ self.operations_progress.update(operation.task_id, visible=False)
548
+
549
+ def update_stats(self, status: Status) -> None:
550
+ """Update statistics for a finished scenario."""
551
+ self.stats[status] += 1
552
+ self._update_stats_display()
553
+
554
+ def interrupt(self) -> None:
555
+ self.is_interrupted = True
556
+ self.stats[Status.SKIP] += self.total - self.current
557
+ if self.live:
558
+ self.stop()
559
+
560
+ def stop(self) -> None:
561
+ """Stop all progress displays."""
562
+ if self.live:
563
+ self.live.stop()
564
+
565
+ def _get_status_icon(self, default_icon: str = "🕛") -> str:
566
+ if self.is_interrupted:
567
+ icon = "⚡"
568
+ elif self.stats[Status.ERROR] > 0:
569
+ icon = "🚫"
570
+ elif self.stats[Status.FAILURE] > 0:
571
+ icon = "❌"
572
+ elif self.stats[Status.SUCCESS] > 0:
573
+ icon = "✅"
574
+ elif self.stats[Status.SKIP] > 0:
575
+ icon = "⏭ "
576
+ else:
577
+ icon = default_icon
578
+ return icon
579
+
580
+ def get_completion_message(self, default_icon: str = "🕛") -> str:
581
+ """Complete the phase and return status message."""
582
+ duration = format_duration(int((time.monotonic() - self.start_time) * 1000))
583
+ icon = self._get_status_icon(default_icon)
584
+
585
+ message = self._get_stats_message() or "No tests were run"
586
+ if self.is_interrupted:
587
+ duration_message = f"interrupted after {duration}"
588
+ else:
589
+ duration_message = f"in {duration}"
590
+
591
+ return f"{icon} {self.title} ({duration_message})\n\n {message}"
592
+
593
+
594
+ @dataclass
595
+ class StatefulProgressManager:
596
+ """Manages progress display for stateful testing."""
597
+
598
+ console: Console
599
+ title: str
600
+ links_selected: int
601
+ links_inferred: int
602
+ links_total: int
603
+ start_time: float
604
+
605
+ # Progress components
606
+ title_progress: Progress
607
+ progress_bar: Progress
608
+ stats_progress: Progress
609
+ live: Live | None
610
+
611
+ # Task IDs
612
+ title_task_id: TaskID | None
613
+ progress_task_id: TaskID | None
614
+ stats_task_id: TaskID
615
+
616
+ # State
617
+ scenarios: int
618
+ links_covered: set[str]
619
+ stats: dict[Status, int]
620
+ is_interrupted: bool
621
+
622
+ __slots__ = (
623
+ "console",
624
+ "title",
625
+ "links_selected",
626
+ "links_inferred",
627
+ "links_total",
628
+ "start_time",
629
+ "title_progress",
630
+ "progress_bar",
631
+ "stats_progress",
632
+ "live",
633
+ "title_task_id",
634
+ "progress_task_id",
635
+ "stats_task_id",
636
+ "scenarios",
637
+ "links_covered",
638
+ "stats",
639
+ "is_interrupted",
640
+ )
641
+
642
+ def __init__(
643
+ self, *, console: Console, title: str, links_selected: int, links_inferred: int, links_total: int
644
+ ) -> None:
645
+ from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
646
+ from rich.style import Style
647
+
648
+ self.console = console
649
+ self.title = title
650
+ self.links_selected = links_selected
651
+ self.links_inferred = links_inferred
652
+ self.links_total = links_total
653
+ self.start_time = time.monotonic()
654
+
655
+ self.title_progress = Progress(
656
+ TextColumn(""),
657
+ SpinnerColumn("clock"),
658
+ TextColumn("{task.description}", style=Style(color="bright_white")),
659
+ console=self.console,
660
+ )
661
+ self.title_task_id = None
662
+
663
+ self.progress_bar = Progress(
664
+ TextColumn(" "),
665
+ TimeElapsedColumn(),
666
+ TextColumn("{task.fields[scenarios]:3d} scenarios • {task.fields[links]}"),
667
+ console=self.console,
668
+ )
669
+ self.progress_task_id = None
670
+
671
+ # Initialize stats progress
672
+ self.stats_progress = Progress(
673
+ TextColumn(" "),
674
+ TextColumn("{task.description}"),
675
+ console=self.console,
676
+ )
677
+ self.stats_task_id = self.stats_progress.add_task("")
678
+
679
+ self.live = None
680
+
681
+ # Initialize state
682
+ self.scenarios = 0
683
+ self.links_covered = set()
684
+ self.stats = {
685
+ Status.SUCCESS: 0,
686
+ Status.FAILURE: 0,
687
+ Status.ERROR: 0,
688
+ Status.SKIP: 0,
689
+ }
690
+ self.is_interrupted = False
691
+
692
+ def start(self) -> None:
693
+ """Start progress display."""
694
+ from rich.console import Group
695
+ from rich.live import Live
696
+ from rich.text import Text
697
+
698
+ # Initialize progress displays
699
+ self.title_task_id = self.title_progress.add_task("Stateful")
700
+ links = f"0 covered / {self.links_selected} selected / {self.links_total} total"
701
+ if self.links_inferred:
702
+ links += f" ({self.links_inferred} inferred)"
703
+ self.progress_task_id = self.progress_bar.add_task("", scenarios=0, links=links)
704
+
705
+ # Create live display
706
+ group = Group(
707
+ self.title_progress,
708
+ Text(),
709
+ self.progress_bar,
710
+ Text(),
711
+ self.stats_progress,
712
+ )
713
+ self.live = Live(group, refresh_per_second=10, console=self.console, transient=True)
714
+ self.live.start()
715
+
716
+ def stop(self) -> None:
717
+ """Stop progress display."""
718
+ if self.live:
719
+ self.live.stop()
720
+
721
+ def update(self, links_covered: set[str], status: Status | None = None) -> None:
722
+ """Update progress and stats."""
723
+ self.scenarios += 1
724
+ self.links_covered.update(links_covered)
725
+
726
+ if status is not None:
727
+ self.stats[status] += 1
728
+
729
+ self._update_progress_display()
730
+ self._update_stats_display()
731
+
732
+ def _update_progress_display(self) -> None:
733
+ """Update the progress display."""
734
+ assert self.progress_task_id is not None
735
+ links = f"{len(self.links_covered)} covered / {self.links_selected} selected / {self.links_total} total"
736
+ if self.links_inferred:
737
+ links += f" ({self.links_inferred} inferred)"
738
+ self.progress_bar.update(self.progress_task_id, scenarios=self.scenarios, links=links)
739
+
740
+ def _get_stats_message(self) -> str:
741
+ """Get formatted stats message."""
742
+ parts = []
743
+ if self.stats[Status.SUCCESS]:
744
+ parts.append(f"✅ {self.stats[Status.SUCCESS]} passed")
745
+ if self.stats[Status.FAILURE]:
746
+ parts.append(f"❌ {self.stats[Status.FAILURE]} failed")
747
+ if self.stats[Status.ERROR]:
748
+ suffix = "s" if self.stats[Status.ERROR] > 1 else ""
749
+ parts.append(f"🚫 {self.stats[Status.ERROR]} error{suffix}")
750
+ if self.stats[Status.SKIP]:
751
+ parts.append(f"⏭ {self.stats[Status.SKIP]} skipped")
752
+ return " ".join(parts)
753
+
754
+ def _update_stats_display(self) -> None:
755
+ """Update the statistics display."""
756
+ self.stats_progress.update(self.stats_task_id, description=self._get_stats_message())
757
+
758
+ def _get_status_icon(self, default_icon: str = "🕛") -> str:
759
+ if self.is_interrupted:
760
+ icon = "⚡"
761
+ elif self.stats[Status.ERROR] > 0:
762
+ icon = "🚫"
763
+ elif self.stats[Status.FAILURE] > 0:
764
+ icon = "❌"
765
+ elif self.stats[Status.SUCCESS] > 0:
766
+ icon = "✅"
767
+ elif self.stats[Status.SKIP] > 0:
768
+ icon = "⏭ "
769
+ else:
770
+ icon = default_icon
771
+ return icon
772
+
773
+ def interrupt(self) -> None:
774
+ """Handle interruption."""
775
+ self.is_interrupted = True
776
+ if self.live:
777
+ self.stop()
778
+
779
+ def get_completion_message(self, icon: str | None = None) -> tuple[str, str]:
780
+ """Complete the phase and return status message."""
781
+ duration = format_duration(int((time.monotonic() - self.start_time) * 1000))
782
+ icon = icon or self._get_status_icon()
783
+
784
+ message = self._get_stats_message() or "No tests were run"
785
+ if self.is_interrupted:
786
+ duration_message = f"interrupted after {duration}"
787
+ else:
788
+ duration_message = f"in {duration}"
789
+
790
+ return f"{icon} {self.title} ({duration_message})", message
791
+
792
+
793
+ def format_duration(duration_ms: int) -> str:
794
+ """Format duration in milliseconds to seconds with 2 decimal places."""
795
+ return f"{duration_ms / 1000:.2f}s"
796
+
797
+
798
+ @dataclass
799
+ class OutputHandler(EventHandler):
800
+ config: ProjectConfig
801
+
802
+ loading_manager: LoadingProgressManager | None = None
803
+ probing_manager: ProbingProgressManager | None = None
804
+ unit_tests_manager: UnitTestProgressManager | None = None
805
+ stateful_tests_manager: StatefulProgressManager | None = None
806
+
807
+ statistic: ApiStatistic | None = None
808
+ skip_reasons: list[str] = field(default_factory=list)
809
+ warnings: WarningData = field(default_factory=WarningData)
810
+ errors: set[events.NonFatalError] = field(default_factory=set)
811
+ phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
812
+ default_factory=lambda: dict.fromkeys(PhaseName, (Status.SKIP, None))
813
+ )
814
+ console: Console = field(default_factory=_default_console)
815
+
816
+ def handle_event(self, ctx: ExecutionContext, event: events.EngineEvent) -> None:
817
+ if isinstance(event, events.PhaseStarted):
818
+ self._on_phase_started(event)
819
+ elif isinstance(event, events.PhaseFinished):
820
+ self._on_phase_finished(event)
821
+ elif isinstance(event, events.ScenarioStarted):
822
+ self._on_scenario_started(event)
823
+ elif isinstance(event, events.ScenarioFinished):
824
+ self._on_scenario_finished(ctx, event)
825
+ elif isinstance(event, events.SchemaAnalysisWarnings):
826
+ self._on_schema_warnings(ctx, event)
827
+ if isinstance(event, events.EngineFinished):
828
+ self._on_engine_finished(ctx, event)
829
+ elif isinstance(event, events.Interrupted):
830
+ self._on_interrupted(event)
831
+ elif isinstance(event, events.FatalError):
832
+ self._on_fatal_error(ctx, event)
833
+ elif isinstance(event, events.NonFatalError):
834
+ self.errors.add(event)
835
+ elif isinstance(event, LoadingStarted):
836
+ self._on_loading_started(event)
837
+ elif isinstance(event, LoadingFinished):
838
+ self._on_loading_finished(ctx, event)
839
+
840
+ def start(self, ctx: ExecutionContext) -> None:
841
+ display_header(SCHEMATHESIS_VERSION)
842
+
843
+ def shutdown(self, ctx: ExecutionContext) -> None:
844
+ if self.unit_tests_manager is not None:
845
+ self.unit_tests_manager.stop()
846
+ if self.stateful_tests_manager is not None:
847
+ self.stateful_tests_manager.stop()
848
+ if self.loading_manager is not None:
849
+ self.loading_manager.stop()
850
+ if self.probing_manager is not None:
851
+ self.probing_manager.stop()
852
+
853
+ def _on_loading_started(self, event: LoadingStarted) -> None:
854
+ self.loading_manager = LoadingProgressManager(console=self.console, location=event.location)
855
+ self.loading_manager.start()
856
+
857
+ def _on_loading_finished(self, ctx: ExecutionContext, event: LoadingFinished) -> None:
858
+ from rich.padding import Padding
859
+ from rich.style import Style
860
+ from rich.table import Table
861
+
862
+ self.config = event.config
863
+
864
+ assert self.loading_manager is not None
865
+ self.loading_manager.stop()
866
+
867
+ message = Padding(
868
+ self.loading_manager.get_completion_message(),
869
+ BLOCK_PADDING,
870
+ )
871
+ self.console.print(message)
872
+ self.console.print()
873
+ self.loading_manager = None
874
+ self.statistic = event.statistic
875
+
876
+ table = Table(
877
+ show_header=False,
878
+ box=None,
879
+ padding=(0, 4),
880
+ collapse_padding=True,
881
+ )
882
+ table.add_column("Field", style=Style(color="bright_white", bold=True))
883
+ table.add_column("Value", style="cyan")
884
+
885
+ table.add_row("Base URL:", event.base_url)
886
+ table.add_row("Specification:", event.specification.name)
887
+ statistic = event.statistic.operations
888
+ table.add_row("Operations:", f"{statistic.selected} selected / {statistic.total} total")
889
+
890
+ message = Padding(table, BLOCK_PADDING)
891
+ self.console.print(message)
892
+ self.console.print()
893
+
894
+ if ctx.initialization_lines:
895
+ _print_lines(ctx.initialization_lines)
896
+
897
+ def _on_phase_started(self, event: events.PhaseStarted) -> None:
898
+ phase = event.phase
899
+ if phase.name == PhaseName.PROBING and phase.is_enabled:
900
+ self._start_probing()
901
+ elif phase.name in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING] and phase.is_enabled:
902
+ self._start_unit_tests(phase.name)
903
+ elif phase.name == PhaseName.STATEFUL_TESTING and phase.is_enabled and phase.skip_reason is None:
904
+ self._start_stateful_tests(event)
905
+
906
+ def _start_probing(self) -> None:
907
+ self.probing_manager = ProbingProgressManager(console=self.console)
908
+ self.probing_manager.start()
909
+
910
+ def _start_unit_tests(self, phase: PhaseName) -> None:
911
+ assert self.statistic is not None
912
+ assert self.unit_tests_manager is None
913
+ self.unit_tests_manager = UnitTestProgressManager(
914
+ console=self.console,
915
+ title=phase.value,
916
+ total=self.statistic.operations.selected,
917
+ )
918
+ self.unit_tests_manager.start()
919
+
920
+ def _start_stateful_tests(self, event: events.PhaseStarted) -> None:
921
+ assert self.statistic is not None
922
+ assert event.payload is not None
923
+ # Total number of links - original ones + inferred during tests
924
+ links_selected = self.statistic.links.selected + event.payload.inferred_links
925
+ links_total = self.statistic.links.total + event.payload.inferred_links
926
+ self.stateful_tests_manager = StatefulProgressManager(
927
+ console=self.console,
928
+ title="Stateful",
929
+ links_selected=links_selected,
930
+ links_inferred=event.payload.inferred_links,
931
+ links_total=links_total,
932
+ )
933
+ self.stateful_tests_manager.start()
934
+
935
+ def _on_phase_finished(self, event: events.PhaseFinished) -> None:
936
+ from rich.padding import Padding
937
+ from rich.style import Style
938
+ from rich.table import Table
939
+ from rich.text import Text
940
+
941
+ phase = event.phase
942
+ self.phases[phase.name] = (event.status, phase.skip_reason)
943
+
944
+ if phase.name == PhaseName.PROBING:
945
+ assert self.probing_manager is not None
946
+ self.probing_manager.stop()
947
+ self.probing_manager = None
948
+
949
+ if event.status == Status.SUCCESS:
950
+ assert isinstance(event.payload, Ok)
951
+ payload = event.payload.ok()
952
+ self.console.print(
953
+ Padding(
954
+ Text.assemble(
955
+ ("✅ ", Style(color="green")),
956
+ ("API capabilities:", Style(color="bright_white")),
957
+ ),
958
+ BLOCK_PADDING,
959
+ )
960
+ )
961
+ self.console.print()
962
+
963
+ table = Table(
964
+ show_header=False,
965
+ box=None,
966
+ padding=(0, 4),
967
+ collapse_padding=True,
968
+ )
969
+ table.add_column("Capability", style=Style(color="bright_white", bold=True))
970
+ table.add_column("Status", style="cyan")
971
+ for probe_run in payload.probes:
972
+ icon, style = {
973
+ ProbeOutcome.SUCCESS: ("✓", Style(color="green")),
974
+ ProbeOutcome.FAILURE: ("✘", Style(color="red")),
975
+ ProbeOutcome.SKIP: ("⊘", Style(color="yellow")),
976
+ }[probe_run.outcome]
977
+
978
+ table.add_row(f"{probe_run.probe.name}:", Text(icon, style=style))
979
+
980
+ message = Padding(table, BLOCK_PADDING)
981
+ else:
982
+ assert event.status == Status.SKIP
983
+ assert isinstance(event.payload, Ok)
984
+ message = Padding(
985
+ Text.assemble(
986
+ ("⏭ ", ""),
987
+ ("API probing skipped", Style(color="yellow")),
988
+ ),
989
+ BLOCK_PADDING,
990
+ )
991
+ self.console.print(message)
992
+ self.console.print()
993
+ elif phase.name == PhaseName.STATEFUL_TESTING and phase.is_enabled and self.stateful_tests_manager is not None:
994
+ self.stateful_tests_manager.stop()
995
+ if event.status == Status.ERROR:
996
+ title, summary = self.stateful_tests_manager.get_completion_message("🚫")
997
+ else:
998
+ title, summary = self.stateful_tests_manager.get_completion_message()
999
+
1000
+ self.console.print(Padding(Text(title, style="bright_white"), BLOCK_PADDING))
1001
+
1002
+ table = Table(
1003
+ show_header=False,
1004
+ box=None,
1005
+ padding=(0, 4),
1006
+ collapse_padding=True,
1007
+ )
1008
+ table.add_column("Field", style=Style(color="bright_white", bold=True))
1009
+ table.add_column("Value", style="cyan")
1010
+ table.add_row("Scenarios:", f"{self.stateful_tests_manager.scenarios}")
1011
+ message = f"{len(self.stateful_tests_manager.links_covered)} covered / {self.stateful_tests_manager.links_selected} selected / {self.stateful_tests_manager.links_total} total"
1012
+ if self.stateful_tests_manager.links_inferred:
1013
+ message += f" ({self.stateful_tests_manager.links_inferred} inferred)"
1014
+ table.add_row("API Links:", message)
1015
+
1016
+ self.console.print()
1017
+ self.console.print(Padding(table, BLOCK_PADDING))
1018
+ self.console.print()
1019
+ self.console.print(Padding(Text(summary, style="bright_white"), (0, 0, 0, 5)))
1020
+ self.console.print()
1021
+ self.stateful_tests_manager = None
1022
+ elif (
1023
+ phase.name in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]
1024
+ and phase.is_enabled
1025
+ and self.unit_tests_manager is not None
1026
+ ):
1027
+ self.unit_tests_manager.stop()
1028
+ if event.status == Status.ERROR:
1029
+ message = self.unit_tests_manager.get_completion_message("🚫")
1030
+ else:
1031
+ message = self.unit_tests_manager.get_completion_message()
1032
+ self.console.print(Padding(Text(message, style="white"), BLOCK_PADDING))
1033
+ self.console.print()
1034
+ self.unit_tests_manager = None
1035
+
1036
+ def _on_scenario_started(self, event: events.ScenarioStarted) -> None:
1037
+ if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
1038
+ # We should display execution result + percentage in the end. For example:
1039
+ assert event.label is not None
1040
+ assert self.unit_tests_manager is not None
1041
+ self.unit_tests_manager.start_operation(event.label)
1042
+
1043
+ def _on_scenario_finished(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1044
+ if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
1045
+ assert self.unit_tests_manager is not None
1046
+ if event.label:
1047
+ self.unit_tests_manager.finish_operation(event.label)
1048
+ self.unit_tests_manager.update_progress()
1049
+ self.unit_tests_manager.update_stats(event.status)
1050
+ if event.status == Status.SKIP and event.skip_reason is not None:
1051
+ self.skip_reasons.append(event.skip_reason)
1052
+ self._check_warnings(ctx, event)
1053
+ elif (
1054
+ event.phase == PhaseName.STATEFUL_TESTING
1055
+ and not event.is_final
1056
+ and event.status not in (Status.INTERRUPTED, Status.SKIP, None)
1057
+ ):
1058
+ assert self.stateful_tests_manager is not None
1059
+ links_seen = {
1060
+ case.transition.id
1061
+ for case in event.recorder.cases.values()
1062
+ if case.transition is not None and case.is_transition_applied
1063
+ }
1064
+ self.stateful_tests_manager.update(links_seen, event.status)
1065
+ self._check_stateful_warnings(ctx, event)
1066
+
1067
+ def _check_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1068
+ from schemathesis.core.compat import RefResolutionError
1069
+
1070
+ statistic = aggregate_status_codes(event.recorder.interactions.values())
1071
+
1072
+ if statistic.total == 0:
1073
+ return
1074
+
1075
+ assert ctx.find_operation_by_label is not None
1076
+ assert event.label is not None
1077
+ try:
1078
+ operation = ctx.find_operation_by_label(event.label)
1079
+ except RefResolutionError:
1080
+ # This error will be reported elsewhere anyway
1081
+ return None
1082
+
1083
+ warnings = self.config.warnings_for(operation=operation)
1084
+
1085
+ def has_only_missing_auth_case() -> bool:
1086
+ case = list(event.recorder.cases.values())[0].value
1087
+ return bool(
1088
+ case.meta
1089
+ and isinstance(case.meta.phase.data, CoveragePhaseData)
1090
+ and case.meta.phase.data.scenario == CoverageScenario.MISSING_PARAMETER
1091
+ and case.meta.phase.data.parameter == "Authorization"
1092
+ and case.meta.phase.data.parameter_location == ParameterLocation.HEADER
1093
+ )
1094
+
1095
+ if warnings.should_display(SchemathesisWarning.MISSING_AUTH):
1096
+ if not (len(event.recorder.cases) == 1 and has_only_missing_auth_case()):
1097
+ for status_code in (401, 403):
1098
+ if statistic.ratio_for(status_code) >= AUTH_ERRORS_THRESHOLD:
1099
+ self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
1100
+ # Check if this warning should cause test failure
1101
+ if warnings.should_fail(SchemathesisWarning.MISSING_AUTH):
1102
+ ctx.exit_code = 1
1103
+
1104
+ # Warn if all positive test cases got 4xx in return and no failure was found
1105
+ def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
1106
+ seen_positive = False
1107
+ for case in recorder.cases.values():
1108
+ if not (case.value.meta is not None and case.value.meta.generation.mode == GenerationMode.POSITIVE):
1109
+ continue
1110
+ seen_positive = True
1111
+ interaction = recorder.interactions.get(case.value.id)
1112
+ if not (interaction is not None and interaction.response is not None):
1113
+ continue
1114
+ # At least one positive response for positive test case
1115
+ if 200 <= interaction.response.status_code < 300:
1116
+ return False
1117
+ # If there are positive test cases, and we ended up here, then there are no 2xx responses for them
1118
+ # Otherwise, there are no positive test cases at all and this check should pass
1119
+ return seen_positive
1120
+
1121
+ if (
1122
+ event.status == Status.SUCCESS
1123
+ and (
1124
+ warnings.should_display(SchemathesisWarning.MISSING_TEST_DATA)
1125
+ or warnings.should_display(SchemathesisWarning.VALIDATION_MISMATCH)
1126
+ )
1127
+ and GenerationMode.POSITIVE in self.config.generation_for(operation=operation, phase=event.phase.name).modes
1128
+ and all_positive_are_rejected(event.recorder)
1129
+ ):
1130
+ if statistic.should_warn_about_missing_test_data():
1131
+ self._handle_warning(
1132
+ ctx,
1133
+ SchemathesisWarning.MISSING_TEST_DATA,
1134
+ lambda: self.warnings.missing_test_data.add(event.recorder.label),
1135
+ )
1136
+ if statistic.should_warn_about_validation_mismatch():
1137
+ self._handle_warning(
1138
+ ctx,
1139
+ SchemathesisWarning.VALIDATION_MISMATCH,
1140
+ lambda: self.warnings.validation_mismatch.add(event.recorder.label),
1141
+ )
1142
+
1143
+ def _handle_warning(
1144
+ self, ctx: ExecutionContext, kind: SchemathesisWarning, record_callback: Callable[[], None]
1145
+ ) -> None:
1146
+ """Handle a warning by checking display/fail config and recording it."""
1147
+ if not self.config.warnings.should_display(kind):
1148
+ return
1149
+ record_callback()
1150
+ if self.config.warnings.should_fail(kind):
1151
+ ctx.exit_code = 1
1152
+
1153
+ def _record_missing_deserializer_warning(self, operation_label: str, message: str) -> Callable[[], None]:
1154
+ """Create a callback that records a missing deserializer warning."""
1155
+
1156
+ def record() -> None:
1157
+ self.warnings.missing_deserializer.setdefault(operation_label, set()).add(message)
1158
+
1159
+ return record
1160
+
1161
+ def _check_stateful_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1162
+ # If stateful testing had successful responses for API operations that were marked with "missing_test_data"
1163
+ # warnings, then remove them from warnings
1164
+ for key, node in event.recorder.cases.items():
1165
+ if not self.warnings.missing_test_data:
1166
+ break
1167
+ if node.value.operation.label in self.warnings.missing_test_data and key in event.recorder.interactions:
1168
+ response = event.recorder.interactions[key].response
1169
+ if response is not None and response.status_code < 300:
1170
+ self.warnings.missing_test_data.remove(node.value.operation.label)
1171
+ continue
1172
+
1173
+ def _on_schema_warnings(self, ctx: ExecutionContext, event: events.SchemaAnalysisWarnings) -> None:
1174
+ """Process schema-level warnings emitted outside of scenarios."""
1175
+ for warning in event.warnings:
1176
+ if warning.kind is SchemathesisWarning.MISSING_DESERIALIZER:
1177
+ self._handle_warning(
1178
+ ctx,
1179
+ warning.kind,
1180
+ self._record_missing_deserializer_warning(warning.operation_label, warning.message),
1181
+ )
1182
+
1183
+ def _on_interrupted(self, event: events.Interrupted) -> None:
1184
+ from rich.padding import Padding
1185
+
1186
+ if self.unit_tests_manager is not None:
1187
+ self.unit_tests_manager.interrupt()
1188
+ elif self.stateful_tests_manager is not None:
1189
+ self.stateful_tests_manager.interrupt()
1190
+ elif self.loading_manager is not None:
1191
+ self.loading_manager.interrupt()
1192
+ message = Padding(
1193
+ self.loading_manager.get_completion_message(),
1194
+ BLOCK_PADDING,
1195
+ )
1196
+ self.console.print(message)
1197
+ self.console.print()
1198
+ elif self.probing_manager is not None:
1199
+ self.probing_manager.interrupt()
1200
+ message = Padding(
1201
+ self.probing_manager.get_completion_message(),
1202
+ BLOCK_PADDING,
1203
+ )
1204
+ self.console.print(message)
1205
+ self.console.print()
1206
+ self.probing_manager = None
1207
+
1208
+ def _on_fatal_error(self, ctx: ExecutionContext, event: events.FatalError) -> None:
1209
+ from rich.padding import Padding
1210
+ from rich.text import Text
1211
+
1212
+ self.shutdown(ctx)
1213
+
1214
+ if isinstance(event.exception, LoaderError):
1215
+ assert self.loading_manager is not None
1216
+ message = Padding(self.loading_manager.get_error_message(event.exception), BLOCK_PADDING)
1217
+ self.console.print(message)
1218
+ self.console.print()
1219
+ self.loading_manager = None
1220
+
1221
+ if event.exception.extras:
1222
+ for extra in event.exception.extras:
1223
+ self.console.print(Padding(Text(extra), (0, 0, 0, 5)))
1224
+ self.console.print()
1225
+
1226
+ if not (
1227
+ event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.config.wait_for_schema is not None
1228
+ ):
1229
+ suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
1230
+ if suggestion is not None:
1231
+ click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
1232
+
1233
+ raise click.Abort
1234
+ title = "Test Execution Error"
1235
+ message = DEFAULT_INTERNAL_ERROR_MESSAGE
1236
+ traceback = format_exception(event.exception, with_traceback=True)
1237
+ extras = split_traceback(traceback)
1238
+ suggestion = f"Please consider reporting the traceback above to our issue tracker:\n\n {ISSUE_TRACKER_URL}."
1239
+ click.echo(_style(title, fg="red", bold=True))
1240
+ click.echo()
1241
+ click.echo(message)
1242
+ _display_extras(extras)
1243
+ if not (
1244
+ isinstance(event.exception, LoaderError)
1245
+ and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
1246
+ and self.config.wait_for_schema is not None
1247
+ ):
1248
+ click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
1249
+
1250
+ raise click.Abort
1251
+
1252
+ def _display_warning_block(
1253
+ self, title: str, operations: set[str] | dict, tips: list[str], operation_suffix: str = ""
1254
+ ) -> None:
1255
+ if isinstance(operations, dict):
1256
+ total = sum(len(ops) for ops in operations.values())
1257
+ else:
1258
+ total = len(operations)
1259
+
1260
+ suffix = "" if total == 1 else "s"
1261
+ click.echo(
1262
+ _style(
1263
+ f"{title}: {total} operation{suffix}{operation_suffix}\n",
1264
+ fg="yellow",
1265
+ )
1266
+ )
1267
+
1268
+ # Print up to 3 endpoints, then "+N more"
1269
+ def _print_up_to_three(operations_: list[str] | set[str]) -> None:
1270
+ for operation in sorted(operations_)[:3]:
1271
+ click.echo(_style(f" - {operation}", fg="yellow"))
1272
+ extra_count = len(operations_) - 3
1273
+ if extra_count > 0:
1274
+ click.echo(_style(f" + {extra_count} more", fg="yellow"))
1275
+
1276
+ if isinstance(operations, dict):
1277
+ for status_code, ops in operations.items():
1278
+ status_text = "Unauthorized" if status_code == 401 else "Forbidden"
1279
+ count = len(ops)
1280
+ suffix = "" if count == 1 else "s"
1281
+ click.echo(_style(f"{status_code} {status_text} ({count} operation{suffix}):", fg="yellow"))
1282
+
1283
+ _print_up_to_three(ops)
1284
+ else:
1285
+ _print_up_to_three(operations)
1286
+
1287
+ if tips:
1288
+ click.echo()
1289
+
1290
+ for tip in tips:
1291
+ click.echo(_style(tip, fg="yellow"))
1292
+
1293
+ click.echo()
1294
+
1295
+ def display_warnings(self) -> None:
1296
+ display_section_name("WARNINGS")
1297
+ click.echo()
1298
+ if self.warnings.missing_auth:
1299
+ self._display_warning_block(
1300
+ title="Authentication failed",
1301
+ operations=self.warnings.missing_auth,
1302
+ operation_suffix=" returned authentication errors",
1303
+ tips=["💡 Ensure valid authentication credentials are set via --auth or -H"],
1304
+ )
1305
+
1306
+ if self.warnings.missing_test_data:
1307
+ self._display_warning_block(
1308
+ title="Missing test data",
1309
+ operations=self.warnings.missing_test_data,
1310
+ operation_suffix=" repeatedly returned 404 Not Found, preventing tests from reaching your API's core logic",
1311
+ tips=[
1312
+ "💡 Provide realistic parameter values in your config file so tests can access existing resources",
1313
+ ],
1314
+ )
1315
+
1316
+ if self.warnings.validation_mismatch:
1317
+ self._display_warning_block(
1318
+ title="Schema validation mismatch",
1319
+ operations=self.warnings.validation_mismatch,
1320
+ operation_suffix=" mostly rejected generated data due to validation errors, indicating schema constraints don't match API validation",
1321
+ tips=["💡 Check your schema constraints - API validation may be stricter than documented"],
1322
+ )
1323
+
1324
+ if self.warnings.missing_deserializer:
1325
+ total = len(self.warnings.missing_deserializer)
1326
+ suffix = "" if total == 1 else "s"
1327
+ click.echo(
1328
+ _style(
1329
+ f"Schema validation skipped: {total} operation{suffix} cannot validate responses due to missing deserializers\n",
1330
+ fg="yellow",
1331
+ )
1332
+ )
1333
+
1334
+ # Show up to 3 operations with their warnings
1335
+ displayed_operations = sorted(self.warnings.missing_deserializer.keys())[:3]
1336
+ for idx, operation_label in enumerate(displayed_operations):
1337
+ messages = self.warnings.missing_deserializer[operation_label]
1338
+ click.echo(_style(f" - {operation_label}", fg="yellow"))
1339
+ for message in sorted(messages):
1340
+ click.echo(_style(f" {message}", fg="yellow"))
1341
+ # Add empty line between operations
1342
+ if idx < len(displayed_operations) - 1:
1343
+ click.echo()
1344
+
1345
+ extra_count = len(self.warnings.missing_deserializer) - 3
1346
+ if extra_count > 0:
1347
+ click.echo(_style(f" + {extra_count} more", fg="yellow"))
1348
+
1349
+ click.echo()
1350
+ click.echo(
1351
+ _style("💡 Register a deserializer with @schemathesis.deserializer() to enable validation", fg="yellow")
1352
+ )
1353
+ click.echo()
1354
+
1355
+ def display_stateful_failures(self, ctx: ExecutionContext) -> None:
1356
+ display_section_name("Stateful tests")
1357
+
1358
+ click.echo("\nFailed to extract data from response:")
1359
+
1360
+ grouped: dict[str, list[ExtractionFailure]] = {}
1361
+ for failure in ctx.statistic.extraction_failures:
1362
+ grouped.setdefault(failure.id, []).append(failure)
1363
+
1364
+ for idx, (transition_id, failures) in enumerate(grouped.items(), 1):
1365
+ for failure in failures:
1366
+ click.echo(f"\n {idx}. Test Case ID: {failure.case_id}\n")
1367
+ click.echo(f" {transition_id}")
1368
+
1369
+ indent = " "
1370
+ if failure.error:
1371
+ if isinstance(failure.error, JSONDecodeError):
1372
+ click.echo(f"\n{indent}Failed to parse JSON from response")
1373
+ else:
1374
+ click.echo(f"\n{indent}{failure.error.__class__.__name__}: {failure.error}")
1375
+ else:
1376
+ if failure.parameter_name == "body":
1377
+ description = f"\n{indent}Could not resolve request body via {failure.expression}"
1378
+ else:
1379
+ description = f"\n{indent}Could not resolve parameter `{failure.parameter_name}` via `{failure.expression}`"
1380
+ prefix = "$response.body"
1381
+ if failure.expression.startswith(prefix):
1382
+ description += f"\n{indent}Path `{failure.expression[len(prefix) :]}` not found in response"
1383
+ click.echo(description)
1384
+
1385
+ click.echo()
1386
+
1387
+ for case, response in reversed(failure.history):
1388
+ curl = case.as_curl_command(headers=dict(response.request.headers), verify=response.verify)
1389
+ click.echo(f"{indent}[{response.status_code}] {curl}")
1390
+
1391
+ response = failure.response
1392
+
1393
+ if response.content is None or not response.content:
1394
+ click.echo(f"\n{indent}<EMPTY>")
1395
+ else:
1396
+ try:
1397
+ payload = prepare_response_payload(response.text, config=ctx.config.output)
1398
+ click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
1399
+ except UnicodeDecodeError:
1400
+ click.echo(f"\n{indent}<BINARY>")
1401
+
1402
+ click.echo()
1403
+
1404
+ def display_api_operations(self, ctx: ExecutionContext) -> None:
1405
+ assert self.statistic is not None
1406
+ click.echo(_style("API Operations:", bold=True))
1407
+ click.echo(
1408
+ _style(
1409
+ f" Selected: {click.style(str(self.statistic.operations.selected), bold=True)}/"
1410
+ f"{click.style(str(self.statistic.operations.total), bold=True)}"
1411
+ )
1412
+ )
1413
+ click.echo(_style(f" Tested: {click.style(str(len(ctx.statistic.tested_operations)), bold=True)}"))
1414
+ errors = len(
1415
+ {
1416
+ err.label
1417
+ for err in self.errors
1418
+ # Some API operations may have some tests before they have an error
1419
+ if err.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]
1420
+ and err.label not in ctx.statistic.tested_operations
1421
+ and err.related_to_operation
1422
+ }
1423
+ )
1424
+ if errors:
1425
+ click.echo(_style(f" Errored: {click.style(str(errors), bold=True)}"))
1426
+
1427
+ # API operations that are skipped due to fail-fast are counted here as well
1428
+ total_skips = self.statistic.operations.selected - len(ctx.statistic.tested_operations) - errors
1429
+ if total_skips:
1430
+ click.echo(_style(f" Skipped: {click.style(str(total_skips), bold=True)}"))
1431
+ for reason in sorted(set(self.skip_reasons)):
1432
+ click.echo(_style(f" - {reason.rstrip('.')}"))
1433
+ click.echo()
1434
+
1435
+ def display_phases(self) -> None:
1436
+ click.echo(_style("Test Phases:", bold=True))
1437
+
1438
+ for phase in PhaseName:
1439
+ if phase in (PhaseName.PROBING, PhaseName.SCHEMA_ANALYSIS):
1440
+ # Internal phases are not part of the test phase summary
1441
+ continue
1442
+ status, skip_reason = self.phases[phase]
1443
+
1444
+ if status == Status.SKIP:
1445
+ click.echo(_style(f" ⏭ {phase.value}", fg="yellow"), nl=False)
1446
+ if skip_reason:
1447
+ click.echo(_style(f" ({skip_reason.value})", fg="yellow"))
1448
+ else:
1449
+ click.echo()
1450
+ elif status == Status.SUCCESS:
1451
+ click.echo(_style(f" ✅ {phase.value}", fg="green"))
1452
+ elif status == Status.FAILURE:
1453
+ click.echo(_style(f" ❌ {phase.value}", fg="red"))
1454
+ elif status == Status.ERROR:
1455
+ click.echo(_style(f" 🚫 {phase.value}", fg="red"))
1456
+ elif status == Status.INTERRUPTED:
1457
+ click.echo(_style(f" ⚡ {phase.value}", fg="yellow"))
1458
+ click.echo()
1459
+
1460
+ def display_test_cases(self, ctx: ExecutionContext) -> None:
1461
+ if ctx.statistic.total_cases == 0:
1462
+ click.echo(_style("Test cases:", bold=True))
1463
+ click.echo(" No test cases were generated\n")
1464
+ return
1465
+
1466
+ unique_failures = sum(
1467
+ len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
1468
+ )
1469
+ click.echo(_style("Test cases:", bold=True))
1470
+
1471
+ parts = [f" {click.style(str(ctx.statistic.total_cases), bold=True)} generated"]
1472
+
1473
+ # Don't show pass/fail status if all cases were skipped
1474
+ if ctx.statistic.cases_without_checks == ctx.statistic.total_cases:
1475
+ parts.append(f"{click.style(str(ctx.statistic.cases_without_checks), bold=True)} skipped")
1476
+ else:
1477
+ if unique_failures > 0:
1478
+ parts.append(
1479
+ f"{click.style(str(ctx.statistic.cases_with_failures), bold=True)} found "
1480
+ f"{click.style(str(unique_failures), bold=True)} unique failures"
1481
+ )
1482
+ else:
1483
+ parts.append(f"{click.style(str(ctx.statistic.total_cases), bold=True)} passed")
1484
+
1485
+ if ctx.statistic.cases_without_checks > 0:
1486
+ parts.append(f"{click.style(str(ctx.statistic.cases_without_checks), bold=True)} skipped")
1487
+
1488
+ click.echo(_style(", ".join(parts) + "\n"))
1489
+
1490
+ def display_failures_summary(self, ctx: ExecutionContext) -> None:
1491
+ # Collect all unique failures and their counts by title
1492
+ failure_counts: dict[str, tuple[Severity, int]] = {}
1493
+ for grouped in ctx.statistic.failures.values():
1494
+ for group in grouped.values():
1495
+ for failure in group.failures:
1496
+ data = failure_counts.get(failure.title, (failure.severity, 0))
1497
+ failure_counts[failure.title] = (failure.severity, data[1] + 1)
1498
+
1499
+ click.echo(_style("Failures:", bold=True))
1500
+
1501
+ # Sort by severity first, then by title
1502
+ sorted_failures = sorted(failure_counts.items(), key=lambda x: (x[1][0], x[0]))
1503
+
1504
+ for title, (_, count) in sorted_failures:
1505
+ click.echo(_style(f" ❌ {title}: "), nl=False)
1506
+ click.echo(_style(str(count), bold=True))
1507
+ click.echo()
1508
+
1509
+ def display_errors_summary(self) -> None:
1510
+ # Group errors by title and count occurrences
1511
+ error_counts: dict[str, int] = {}
1512
+ for error in self.errors:
1513
+ title = error.info.title
1514
+ error_counts[title] = error_counts.get(title, 0) + 1
1515
+
1516
+ click.echo(_style("Errors:", bold=True))
1517
+
1518
+ for title in sorted(error_counts):
1519
+ click.echo(_style(f" 🚫 {title}: "), nl=False)
1520
+ click.echo(_style(str(error_counts[title]), bold=True))
1521
+ click.echo()
1522
+
1523
+ def display_final_line(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
1524
+ parts = []
1525
+
1526
+ unique_failures = sum(
1527
+ len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
1528
+ )
1529
+ if unique_failures:
1530
+ suffix = "s" if unique_failures > 1 else ""
1531
+ parts.append(f"{unique_failures} failure{suffix}")
1532
+
1533
+ if self.errors:
1534
+ suffix = "s" if len(self.errors) > 1 else ""
1535
+ parts.append(f"{len(self.errors)} error{suffix}")
1536
+
1537
+ total_warnings = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
1538
+ if total_warnings:
1539
+ suffix = "s" if total_warnings > 1 else ""
1540
+ parts.append(f"{total_warnings} warning{suffix}")
1541
+
1542
+ if parts:
1543
+ message = f"{', '.join(parts)} in {event.running_time:.2f}s"
1544
+ color = "red" if (unique_failures or self.errors) else "yellow"
1545
+ elif ctx.statistic.total_cases == 0:
1546
+ message = "Empty test suite"
1547
+ color = "yellow"
1548
+ else:
1549
+ message = f"No issues found in {event.running_time:.2f}s"
1550
+ color = "green"
1551
+
1552
+ display_section_name(message, fg=color)
1553
+
1554
+ def display_reports(self) -> None:
1555
+ reports = self.config.reports
1556
+ if reports.vcr.enabled or reports.har.enabled or reports.junit.enabled:
1557
+ click.echo(_style("Reports:", bold=True))
1558
+ for format, report in (
1559
+ (ReportFormat.JUNIT, reports.junit),
1560
+ (ReportFormat.VCR, reports.vcr),
1561
+ (ReportFormat.HAR, reports.har),
1562
+ ):
1563
+ if report.enabled:
1564
+ path = reports.get_path(format)
1565
+ click.echo(_style(f" - {format.value.upper()}: {path}"))
1566
+ click.echo()
1567
+
1568
+ def display_seed(self) -> None:
1569
+ click.echo(_style("Seed: ", bold=True), nl=False)
1570
+ # Deterministic mode can be applied to a subset of tests, but we only care if it is enabled everywhere
1571
+ # If not everywhere, then the seed matter and should be displayed
1572
+ if self.config.seed is None or self.config.generation.deterministic:
1573
+ click.echo("not used in the deterministic mode")
1574
+ else:
1575
+ click.echo(str(self.config.seed))
1576
+ click.echo()
1577
+
1578
+ def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
1579
+ assert self.loading_manager is None
1580
+ assert self.probing_manager is None
1581
+ assert self.unit_tests_manager is None
1582
+ assert self.stateful_tests_manager is None
1583
+ if self.errors:
1584
+ display_section_name("ERRORS")
1585
+ errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
1586
+ for label, group_errors in groupby(errors, key=lambda r: r.label):
1587
+ display_section_name(label, "_", fg="red")
1588
+ _errors = list(group_errors)
1589
+ for idx, error in enumerate(_errors, 1):
1590
+ click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1591
+ if idx < len(_errors):
1592
+ click.echo()
1593
+ click.echo(
1594
+ _style(
1595
+ f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
1596
+ fg="red",
1597
+ )
1598
+ )
1599
+ display_failures(ctx)
1600
+ if not self.warnings.is_empty:
1601
+ self.display_warnings()
1602
+ if ctx.statistic.extraction_failures:
1603
+ self.display_stateful_failures(ctx)
1604
+ display_section_name("SUMMARY")
1605
+ click.echo()
1606
+
1607
+ if self.statistic:
1608
+ self.display_api_operations(ctx)
1609
+
1610
+ self.display_phases()
1611
+
1612
+ if ctx.statistic.failures:
1613
+ self.display_failures_summary(ctx)
1614
+
1615
+ if self.errors:
1616
+ self.display_errors_summary()
1617
+
1618
+ if not self.warnings.is_empty:
1619
+ click.echo(_style("Warnings:", bold=True))
1620
+
1621
+ if self.warnings.missing_auth:
1622
+ affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
1623
+ suffix = "" if affected == 1 else "s"
1624
+ click.echo(
1625
+ _style(
1626
+ f" ⚠️ Missing authentication: {bold(str(affected))} operation{suffix} returned only 401/403 responses",
1627
+ fg="yellow",
1628
+ )
1629
+ )
1630
+
1631
+ if self.warnings.missing_test_data:
1632
+ count = len(self.warnings.missing_test_data)
1633
+ suffix = "" if count == 1 else "s"
1634
+ click.echo(
1635
+ _style(
1636
+ f" ⚠️ Missing valid test data: {bold(str(count))} operation{suffix} repeatedly returned 404 responses",
1637
+ fg="yellow",
1638
+ )
1639
+ )
1640
+
1641
+ if self.warnings.validation_mismatch:
1642
+ count = len(self.warnings.validation_mismatch)
1643
+ suffix = "" if count == 1 else "s"
1644
+ click.echo(
1645
+ _style(
1646
+ f" ⚠️ Schema validation mismatch: {bold(str(count))} operation{suffix} mostly rejected generated data",
1647
+ fg="yellow",
1648
+ )
1649
+ )
1650
+
1651
+ if self.warnings.missing_deserializer:
1652
+ count = len(self.warnings.missing_deserializer)
1653
+ suffix = "" if count == 1 else "s"
1654
+ click.echo(
1655
+ _style(
1656
+ f" ⚠️ Schema validation skipped: {bold(str(count))} operation{suffix} cannot validate responses",
1657
+ fg="yellow",
1658
+ )
1659
+ )
1660
+
1661
+ click.echo()
1662
+
1663
+ if ctx.summary_lines:
1664
+ _print_lines(ctx.summary_lines)
1665
+ click.echo()
1666
+
1667
+ self.display_test_cases(ctx)
1668
+ self.display_reports()
1669
+ self.display_seed()
1670
+ self.display_final_line(ctx, event)
1671
+
1672
+
1673
+ @dataclass
1674
+ class StatusCodeStatistic:
1675
+ """Statistics about HTTP status codes in a scenario."""
1676
+
1677
+ counts: dict[int, int]
1678
+ total: int
1679
+
1680
+ __slots__ = ("counts", "total")
1681
+
1682
+ def ratio_for(self, status_code: int) -> float:
1683
+ """Calculate the ratio of responses with the given status code."""
1684
+ if self.total == 0:
1685
+ return 0.0
1686
+ return self.counts.get(status_code, 0) / self.total
1687
+
1688
+ def _get_4xx_breakdown(self) -> tuple[int, int, int]:
1689
+ """Get breakdown of 4xx responses: (404_count, other_4xx_count, total_4xx_count)."""
1690
+ count_404 = self.counts.get(404, 0)
1691
+ count_other_4xx = sum(
1692
+ count for code, count in self.counts.items() if 400 <= code < 500 and code not in {401, 403, 404}
1693
+ )
1694
+ total_4xx = count_404 + count_other_4xx
1695
+ return count_404, count_other_4xx, total_4xx
1696
+
1697
+ def _is_only_4xx_responses(self) -> bool:
1698
+ """Check if all responses are 4xx (excluding 5xx)."""
1699
+ return all(400 <= code < 500 for code in self.counts.keys() if code not in {500})
1700
+
1701
+ def _can_warn_about_4xx(self) -> bool:
1702
+ """Check basic conditions for 4xx warnings."""
1703
+ if self.total == 0:
1704
+ return False
1705
+ # Skip if only auth errors
1706
+ if set(self.counts.keys()) <= {401, 403, 500}:
1707
+ return False
1708
+ return self._is_only_4xx_responses()
1709
+
1710
+ def should_warn_about_missing_test_data(self) -> bool:
1711
+ """Check if an operation should be warned about missing test data (significant 404 responses)."""
1712
+ if not self._can_warn_about_4xx():
1713
+ return False
1714
+
1715
+ count_404, _, total_4xx = self._get_4xx_breakdown()
1716
+
1717
+ if total_4xx == 0:
1718
+ return False
1719
+
1720
+ return (count_404 / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1721
+
1722
+ def should_warn_about_validation_mismatch(self) -> bool:
1723
+ """Check if an operation should be warned about validation mismatch (significant 400/422 responses)."""
1724
+ if not self._can_warn_about_4xx():
1725
+ return False
1726
+
1727
+ _, count_other_4xx, total_4xx = self._get_4xx_breakdown()
1728
+
1729
+ if total_4xx == 0:
1730
+ return False
1731
+
1732
+ return (count_other_4xx / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1733
+
1734
+
1735
+ AUTH_ERRORS_THRESHOLD = 0.9
1736
+ OTHER_CLIENT_ERRORS_THRESHOLD = 0.1
1737
+
1738
+
1739
+ def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
1740
+ """Analyze status codes from interactions."""
1741
+ counts: dict[int, int] = {}
1742
+ total = 0
1743
+
1744
+ for interaction in interactions:
1745
+ if interaction.response is not None:
1746
+ status = interaction.response.status_code
1747
+ counts[status] = counts.get(status, 0) + 1
1748
+ total += 1
1749
+
1750
+ return StatusCodeStatistic(counts=counts, total=total)