schemathesis 4.0.0a2__py3-none-any.whl → 4.0.0a4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. schemathesis/cli/__init__.py +15 -4
  2. schemathesis/cli/commands/run/__init__.py +148 -94
  3. schemathesis/cli/commands/run/context.py +72 -2
  4. schemathesis/cli/commands/run/events.py +22 -2
  5. schemathesis/cli/commands/run/executor.py +35 -12
  6. schemathesis/cli/commands/run/filters.py +1 -0
  7. schemathesis/cli/commands/run/handlers/cassettes.py +27 -46
  8. schemathesis/cli/commands/run/handlers/junitxml.py +1 -1
  9. schemathesis/cli/commands/run/handlers/output.py +180 -87
  10. schemathesis/cli/commands/run/hypothesis.py +30 -19
  11. schemathesis/cli/commands/run/reports.py +72 -0
  12. schemathesis/cli/commands/run/validation.py +18 -12
  13. schemathesis/cli/ext/groups.py +42 -13
  14. schemathesis/cli/ext/options.py +15 -8
  15. schemathesis/core/errors.py +85 -9
  16. schemathesis/core/failures.py +2 -1
  17. schemathesis/core/transforms.py +1 -1
  18. schemathesis/engine/core.py +1 -1
  19. schemathesis/engine/errors.py +17 -6
  20. schemathesis/engine/phases/stateful/__init__.py +1 -0
  21. schemathesis/engine/phases/stateful/_executor.py +9 -12
  22. schemathesis/engine/phases/unit/__init__.py +2 -3
  23. schemathesis/engine/phases/unit/_executor.py +16 -13
  24. schemathesis/engine/recorder.py +22 -21
  25. schemathesis/errors.py +23 -13
  26. schemathesis/filters.py +8 -0
  27. schemathesis/generation/coverage.py +10 -5
  28. schemathesis/generation/hypothesis/builder.py +15 -12
  29. schemathesis/generation/stateful/state_machine.py +57 -12
  30. schemathesis/pytest/lazy.py +2 -3
  31. schemathesis/pytest/plugin.py +2 -3
  32. schemathesis/schemas.py +1 -1
  33. schemathesis/specs/openapi/checks.py +77 -37
  34. schemathesis/specs/openapi/expressions/__init__.py +22 -6
  35. schemathesis/specs/openapi/expressions/nodes.py +15 -21
  36. schemathesis/specs/openapi/expressions/parser.py +1 -1
  37. schemathesis/specs/openapi/parameters.py +0 -2
  38. schemathesis/specs/openapi/patterns.py +170 -2
  39. schemathesis/specs/openapi/schemas.py +67 -39
  40. schemathesis/specs/openapi/stateful/__init__.py +207 -84
  41. schemathesis/specs/openapi/stateful/control.py +87 -0
  42. schemathesis/specs/openapi/{links.py → stateful/links.py} +72 -14
  43. {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/METADATA +1 -1
  44. {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/RECORD +47 -45
  45. {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/WHEEL +0 -0
  46. {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/entry_points.txt +0 -0
  47. {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/licenses/LICENSE +0 -0
@@ -1,8 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import os
4
+ import textwrap
4
5
  import time
5
6
  from dataclasses import dataclass, field
7
+ from json.decoder import JSONDecodeError
6
8
  from types import GeneratorType
7
9
  from typing import TYPE_CHECKING, Any, Generator, Iterable
8
10
 
@@ -11,11 +13,12 @@ import click
11
13
  from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
12
14
  from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
13
15
  from schemathesis.cli.commands.run.handlers.base import EventHandler
14
- from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig
16
+ from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
15
17
  from schemathesis.cli.constants import ISSUE_TRACKER_URL
16
18
  from schemathesis.cli.core import get_terminal_width
17
19
  from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
18
20
  from schemathesis.core.failures import MessageBlock, Severity, format_failures
21
+ from schemathesis.core.output import prepare_response_payload
19
22
  from schemathesis.core.result import Err, Ok
20
23
  from schemathesis.core.version import SCHEMATHESIS_VERSION
21
24
  from schemathesis.engine import Status, events
@@ -32,6 +35,8 @@ if TYPE_CHECKING:
32
35
  from rich.progress import Progress, TaskID
33
36
  from rich.text import Text
34
37
 
38
+ from schemathesis.generation.stateful.state_machine import ExtractionFailure
39
+
35
40
  IO_ENCODING = os.getenv("PYTHONIOENCODING", "utf-8")
36
41
  DISCORD_LINK = "https://discord.gg/R9ASRAmHnA"
37
42
 
@@ -40,7 +45,7 @@ def display_section_name(title: str, separator: str = "=", **kwargs: Any) -> Non
40
45
  """Print section name with separators in terminal with the given title nicely centered."""
41
46
  message = f" {title} ".center(get_terminal_width(), separator)
42
47
  kwargs.setdefault("bold", True)
43
- click.secho(message, **kwargs)
48
+ click.echo(_style(message, **kwargs))
44
49
 
45
50
 
46
51
  def bold(option: str) -> str:
@@ -58,12 +63,14 @@ def display_failures(ctx: ExecutionContext) -> None:
58
63
 
59
64
 
60
65
  if IO_ENCODING != "utf-8":
66
+ HEADER_SEPARATOR = "-"
61
67
 
62
68
  def _style(text: str, **kwargs: Any) -> str:
63
69
  text = text.encode(IO_ENCODING, errors="replace").decode("utf-8")
64
70
  return click.style(text, **kwargs)
65
71
 
66
72
  else:
73
+ HEADER_SEPARATOR = "━"
67
74
 
68
75
  def _style(text: str, **kwargs: Any) -> str:
69
76
  return click.style(text, **kwargs)
@@ -98,7 +105,7 @@ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks:
98
105
 
99
106
 
100
107
  VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema or GraphQL endpoint"
101
- DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}."
108
+ DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--tls-verify=false`')}."
102
109
  LOADER_ERROR_SUGGESTIONS = {
103
110
  # SSL-specific connection issue
104
111
  LoaderErrorKind.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
@@ -122,14 +129,14 @@ def _display_extras(extras: list[str]) -> None:
122
129
  if extras:
123
130
  click.echo()
124
131
  for extra in extras:
125
- click.secho(f" {extra}")
132
+ click.echo(_style(f" {extra}"))
126
133
 
127
134
 
128
135
  def display_header(version: str) -> None:
129
136
  prefix = "v" if version != "dev" else ""
130
137
  header = f"Schemathesis {prefix}{version}"
131
- click.secho(header, bold=True)
132
- click.secho("━" * len(header), bold=True)
138
+ click.echo(_style(header, bold=True))
139
+ click.echo(_style(HEADER_SEPARATOR * len(header), bold=True))
133
140
  click.echo()
134
141
 
135
142
 
@@ -377,6 +384,7 @@ class UnitTestProgressManager:
377
384
 
378
385
  def __init__(
379
386
  self,
387
+ *,
380
388
  console: Console,
381
389
  title: str,
382
390
  total: int,
@@ -435,6 +443,7 @@ class UnitTestProgressManager:
435
443
  Status.FAILURE: 0,
436
444
  Status.SKIP: 0,
437
445
  Status.ERROR: 0,
446
+ Status.INTERRUPTED: 0,
438
447
  }
439
448
  self._update_stats_display()
440
449
 
@@ -451,8 +460,8 @@ class UnitTestProgressManager:
451
460
  parts.append(f"❌ {self.stats[Status.FAILURE]:{width}d} failed")
452
461
  if self.stats[Status.ERROR]:
453
462
  parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} errors")
454
- if self.stats[Status.SKIP]:
455
- parts.append(f"⏭️ {self.stats[Status.SKIP]:{width}d} skipped")
463
+ if self.stats[Status.SKIP] or self.stats[Status.INTERRUPTED]:
464
+ parts.append(f"⏭️ {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
456
465
  return " ".join(parts)
457
466
 
458
467
  def _update_stats_display(self) -> None:
@@ -558,6 +567,7 @@ class StatefulProgressManager:
558
567
 
559
568
  console: Console
560
569
  title: str
570
+ links_selected: int
561
571
  links_total: int
562
572
  start_time: float
563
573
 
@@ -574,13 +584,14 @@ class StatefulProgressManager:
574
584
 
575
585
  # State
576
586
  scenarios: int
577
- links_seen: set[str]
587
+ links_covered: set[str]
578
588
  stats: dict[Status, int]
579
589
  is_interrupted: bool
580
590
 
581
591
  __slots__ = (
582
592
  "console",
583
593
  "title",
594
+ "links_selected",
584
595
  "links_total",
585
596
  "start_time",
586
597
  "title_progress",
@@ -591,17 +602,18 @@ class StatefulProgressManager:
591
602
  "progress_task_id",
592
603
  "stats_task_id",
593
604
  "scenarios",
594
- "links_seen",
605
+ "links_covered",
595
606
  "stats",
596
607
  "is_interrupted",
597
608
  )
598
609
 
599
- def __init__(self, console: Console, title: str, links_total: int) -> None:
610
+ def __init__(self, *, console: Console, title: str, links_selected: int, links_total: int) -> None:
600
611
  from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
601
612
  from rich.style import Style
602
613
 
603
614
  self.console = console
604
615
  self.title = title
616
+ self.links_selected = links_selected
605
617
  self.links_total = links_total
606
618
  self.start_time = time.monotonic()
607
619
 
@@ -633,7 +645,7 @@ class StatefulProgressManager:
633
645
 
634
646
  # Initialize state
635
647
  self.scenarios = 0
636
- self.links_seen = set()
648
+ self.links_covered = set()
637
649
  self.stats = {
638
650
  Status.SUCCESS: 0,
639
651
  Status.FAILURE: 0,
@@ -650,7 +662,9 @@ class StatefulProgressManager:
650
662
 
651
663
  # Initialize progress displays
652
664
  self.title_task_id = self.title_progress.add_task("Stateful tests")
653
- self.progress_task_id = self.progress_bar.add_task("", scenarios=0, links=f"0/{self.links_total} links")
665
+ self.progress_task_id = self.progress_bar.add_task(
666
+ "", scenarios=0, links=f"0 covered / {self.links_selected} selected / {self.links_total} total links"
667
+ )
654
668
 
655
669
  # Create live display
656
670
  group = Group(
@@ -668,10 +682,10 @@ class StatefulProgressManager:
668
682
  if self.live:
669
683
  self.live.stop()
670
684
 
671
- def update(self, links_seen: set[str], status: Status | None = None) -> None:
685
+ def update(self, links_covered: set[str], status: Status | None = None) -> None:
672
686
  """Update progress and stats."""
673
687
  self.scenarios += 1
674
- self.links_seen.update(links_seen)
688
+ self.links_covered.update(links_covered)
675
689
 
676
690
  if status is not None:
677
691
  self.stats[status] += 1
@@ -685,7 +699,7 @@ class StatefulProgressManager:
685
699
  self.progress_bar.update(
686
700
  self.progress_task_id,
687
701
  scenarios=self.scenarios,
688
- links=f"{len(self.links_seen)}/{self.links_total} links",
702
+ links=f"{len(self.links_covered)} covered / {self.links_selected} selected / {self.links_total} total links",
689
703
  )
690
704
 
691
705
  def _get_stats_message(self) -> str:
@@ -770,6 +784,8 @@ def format_duration(duration_ms: int) -> str:
770
784
  @dataclass
771
785
  class OutputHandler(EventHandler):
772
786
  workers_num: int
787
+ # Seed can't be absent in the deterministic mode
788
+ seed: int | None
773
789
  rate_limit: str | None
774
790
  wait_for_schema: float | None
775
791
 
@@ -780,8 +796,7 @@ class OutputHandler(EventHandler):
780
796
 
781
797
  statistic: ApiStatistic | None = None
782
798
  skip_reasons: list[str] = field(default_factory=list)
783
- cassette_config: CassetteConfig | None = None
784
- junit_xml_file: str | None = None
799
+ report_config: ReportConfig | None = None
785
800
  warnings: WarningData = field(default_factory=WarningData)
786
801
  errors: list[events.NonFatalError] = field(default_factory=list)
787
802
  phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
@@ -856,7 +871,8 @@ class OutputHandler(EventHandler):
856
871
 
857
872
  table.add_row("Base URL:", event.base_url)
858
873
  table.add_row("Specification:", event.specification.name)
859
- table.add_row("Operations:", str(event.statistic.operations.total))
874
+ statistic = event.statistic.operations
875
+ table.add_row("Operations:", f"{statistic.selected} selected / {statistic.total} total")
860
876
 
861
877
  message = Padding(table, BLOCK_PADDING)
862
878
  self.console.print(message)
@@ -883,7 +899,7 @@ class OutputHandler(EventHandler):
883
899
  self.unit_tests_manager = UnitTestProgressManager(
884
900
  console=self.console,
885
901
  title="Unit tests",
886
- total=self.statistic.operations.total,
902
+ total=self.statistic.operations.selected,
887
903
  )
888
904
  self.unit_tests_manager.start()
889
905
 
@@ -892,6 +908,7 @@ class OutputHandler(EventHandler):
892
908
  self.stateful_tests_manager = StatefulProgressManager(
893
909
  console=self.console,
894
910
  title="Stateful tests",
911
+ links_selected=self.statistic.links.selected,
895
912
  links_total=self.statistic.links.total,
896
913
  )
897
914
  self.stateful_tests_manager.start()
@@ -984,7 +1001,8 @@ class OutputHandler(EventHandler):
984
1001
  table.add_column("Value", style="cyan")
985
1002
  table.add_row("Scenarios:", f"{self.stateful_tests_manager.scenarios}")
986
1003
  table.add_row(
987
- "API Links:", f"{len(self.stateful_tests_manager.links_seen)}/{self.stateful_tests_manager.links_total}"
1004
+ "API Links:",
1005
+ f"{len(self.stateful_tests_manager.links_covered)} covered / {self.stateful_tests_manager.links_selected} selected / {self.stateful_tests_manager.links_total} total",
988
1006
  )
989
1007
 
990
1008
  self.console.print()
@@ -1081,7 +1099,7 @@ class OutputHandler(EventHandler):
1081
1099
  if not (event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.wait_for_schema is not None):
1082
1100
  suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
1083
1101
  if suggestion is not None:
1084
- click.secho(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}")
1102
+ click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
1085
1103
 
1086
1104
  raise click.Abort
1087
1105
  title = "Test Execution Error"
@@ -1089,16 +1107,16 @@ class OutputHandler(EventHandler):
1089
1107
  traceback = format_exception(event.exception, with_traceback=True)
1090
1108
  extras = split_traceback(traceback)
1091
1109
  suggestion = f"Please consider reporting the traceback above to our issue tracker:\n\n {ISSUE_TRACKER_URL}."
1092
- click.secho(title, fg="red", bold=True)
1110
+ click.echo(_style(title, fg="red", bold=True))
1093
1111
  click.echo()
1094
- click.secho(message)
1112
+ click.echo(message)
1095
1113
  _display_extras(extras)
1096
1114
  if not (
1097
1115
  isinstance(event.exception, LoaderError)
1098
1116
  and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
1099
1117
  and self.wait_for_schema is not None
1100
1118
  ):
1101
- click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}")
1119
+ click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
1102
1120
 
1103
1121
  raise click.Abort
1104
1122
 
@@ -1106,29 +1124,33 @@ class OutputHandler(EventHandler):
1106
1124
  display_section_name("WARNINGS")
1107
1125
  total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
1108
1126
  suffix = "" if total == 1 else "s"
1109
- click.secho(
1110
- f"\nMissing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
1111
- fg="yellow",
1127
+ click.echo(
1128
+ _style(
1129
+ f"\nMissing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
1130
+ fg="yellow",
1131
+ )
1112
1132
  )
1113
1133
 
1114
1134
  for status_code, operations in self.warnings.missing_auth.items():
1115
1135
  status_text = "Unauthorized" if status_code == 401 else "Forbidden"
1116
1136
  count = len(operations)
1117
1137
  suffix = "" if count == 1 else "s"
1118
- click.secho(
1119
- f"{status_code} {status_text} ({count} operation{suffix}):",
1120
- fg="yellow",
1138
+ click.echo(
1139
+ _style(
1140
+ f"{status_code} {status_text} ({count} operation{suffix}):",
1141
+ fg="yellow",
1142
+ )
1121
1143
  )
1122
1144
  # Show first few API operations
1123
1145
  for endpoint in operations[:3]:
1124
- click.secho(f" {endpoint}", fg="yellow")
1146
+ click.echo(_style(f" - {endpoint}", fg="yellow"))
1125
1147
  if len(operations) > 3:
1126
- click.secho(f" + {len(operations) - 3} more", fg="yellow")
1148
+ click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
1127
1149
  click.echo()
1128
- click.secho("Tip: ", bold=True, fg="yellow", nl=False)
1129
- click.secho(f"Use {bold('--auth')} ", fg="yellow", nl=False)
1130
- click.secho(f"or {bold('-H')} ", fg="yellow", nl=False)
1131
- click.secho("to provide authentication credentials", fg="yellow")
1150
+ click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
1151
+ click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
1152
+ click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
1153
+ click.echo(_style("to provide authentication credentials", fg="yellow"))
1132
1154
  click.echo()
1133
1155
 
1134
1156
  def display_experiments(self) -> None:
@@ -1136,26 +1158,78 @@ class OutputHandler(EventHandler):
1136
1158
 
1137
1159
  click.echo()
1138
1160
  for experiment in sorted(GLOBAL_EXPERIMENTS.enabled, key=lambda e: e.name):
1139
- click.secho(f"🧪 {experiment.name}: ", bold=True, nl=False)
1140
- click.secho(experiment.description)
1141
- click.secho(f" Feedback: {experiment.discussion_url}")
1161
+ click.echo(_style(f"🧪 {experiment.name}: ", bold=True), nl=False)
1162
+ click.echo(_style(experiment.description))
1163
+ click.echo(_style(f" Feedback: {experiment.discussion_url}"))
1142
1164
  click.echo()
1143
1165
 
1144
- click.secho(
1145
- "Your feedback is crucial for experimental features. "
1146
- "Please visit the provided URL(s) to share your thoughts.",
1147
- dim=True,
1166
+ click.echo(
1167
+ _style(
1168
+ "Your feedback is crucial for experimental features. "
1169
+ "Please visit the provided URL(s) to share your thoughts.",
1170
+ dim=True,
1171
+ )
1148
1172
  )
1149
1173
  click.echo()
1150
1174
 
1175
+ def display_stateful_failures(self, ctx: ExecutionContext) -> None:
1176
+ display_section_name("Stateful tests")
1177
+
1178
+ click.echo("\nFailed to extract data from response:")
1179
+
1180
+ grouped: dict[str, list[ExtractionFailure]] = {}
1181
+ for failure in ctx.statistic.extraction_failures:
1182
+ grouped.setdefault(failure.id, []).append(failure)
1183
+
1184
+ for idx, (transition_id, failures) in enumerate(grouped.items(), 1):
1185
+ for failure in failures:
1186
+ click.echo(f"\n {idx}. Test Case ID: {failure.case_id}\n")
1187
+ click.echo(f" {transition_id}")
1188
+
1189
+ indent = " "
1190
+ if failure.error:
1191
+ if isinstance(failure.error, JSONDecodeError):
1192
+ click.echo(f"\n{indent}Failed to parse JSON from response")
1193
+ else:
1194
+ click.echo(f"\n{indent}{failure.error.__class__.__name__}: {failure.error}")
1195
+ else:
1196
+ description = (
1197
+ f"\n{indent}Could not resolve parameter `{failure.parameter_name}` via `{failure.expression}`"
1198
+ )
1199
+ prefix = "$response.body"
1200
+ if failure.expression.startswith(prefix):
1201
+ description += f"\n{indent}Path `{failure.expression[len(prefix) :]}` not found in response"
1202
+ click.echo(description)
1203
+
1204
+ click.echo()
1205
+
1206
+ for case, response in reversed(failure.history):
1207
+ curl = case.as_curl_command(headers=dict(response.request.headers), verify=response.verify)
1208
+ click.echo(f"{indent}[{response.status_code}] {curl}")
1209
+
1210
+ response = failure.response
1211
+
1212
+ if response.content is None or not response.content:
1213
+ click.echo(f"\n{indent}<EMPTY>")
1214
+ else:
1215
+ try:
1216
+ payload = prepare_response_payload(response.text, config=ctx.output_config)
1217
+ click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
1218
+ except UnicodeDecodeError:
1219
+ click.echo(f"\n{indent}<BINARY>")
1220
+
1221
+ click.echo()
1222
+
1151
1223
  def display_api_operations(self, ctx: ExecutionContext) -> None:
1152
1224
  assert self.statistic is not None
1153
- click.secho("API Operations:", bold=True)
1154
- click.secho(
1155
- f" Selected: {click.style(str(self.statistic.operations.selected), bold=True)}/"
1156
- f"{click.style(str(self.statistic.operations.total), bold=True)}"
1225
+ click.echo(_style("API Operations:", bold=True))
1226
+ click.echo(
1227
+ _style(
1228
+ f" Selected: {click.style(str(self.statistic.operations.selected), bold=True)}/"
1229
+ f"{click.style(str(self.statistic.operations.total), bold=True)}"
1230
+ )
1157
1231
  )
1158
- click.secho(f" Tested: {click.style(str(len(ctx.statistic.tested_operations)), bold=True)}")
1232
+ click.echo(_style(f" Tested: {click.style(str(len(ctx.statistic.tested_operations)), bold=True)}"))
1159
1233
  errors = len(
1160
1234
  {
1161
1235
  err.label
@@ -1167,48 +1241,48 @@ class OutputHandler(EventHandler):
1167
1241
  }
1168
1242
  )
1169
1243
  if errors:
1170
- click.secho(f" Errored: {click.style(str(errors), bold=True)}")
1244
+ click.echo(_style(f" Errored: {click.style(str(errors), bold=True)}"))
1171
1245
 
1172
1246
  # API operations that are skipped due to fail-fast are counted here as well
1173
1247
  total_skips = self.statistic.operations.selected - len(ctx.statistic.tested_operations) - errors
1174
1248
  if total_skips:
1175
- click.secho(f" Skipped: {click.style(str(total_skips), bold=True)}")
1249
+ click.echo(_style(f" Skipped: {click.style(str(total_skips), bold=True)}"))
1176
1250
  for reason in sorted(set(self.skip_reasons)):
1177
- click.secho(f" - {reason.rstrip('.')}")
1251
+ click.echo(_style(f" - {reason.rstrip('.')}"))
1178
1252
  click.echo()
1179
1253
 
1180
1254
  def display_phases(self) -> None:
1181
- click.secho("Test Phases:", bold=True)
1255
+ click.echo(_style("Test Phases:", bold=True))
1182
1256
 
1183
1257
  for phase in PhaseName:
1184
1258
  status, skip_reason = self.phases[phase]
1185
1259
 
1186
1260
  if status == Status.SKIP:
1187
- click.secho(f" ⏭️ {phase.value}", fg="yellow", nl=False)
1261
+ click.echo(_style(f" ⏭️ {phase.value}", fg="yellow"), nl=False)
1188
1262
  if skip_reason:
1189
- click.secho(f" ({skip_reason.value})", fg="yellow")
1263
+ click.echo(_style(f" ({skip_reason.value})", fg="yellow"))
1190
1264
  else:
1191
1265
  click.echo()
1192
1266
  elif status == Status.SUCCESS:
1193
- click.secho(f" ✅ {phase.value}", fg="green")
1267
+ click.echo(_style(f" ✅ {phase.value}", fg="green"))
1194
1268
  elif status == Status.FAILURE:
1195
- click.secho(f" ❌ {phase.value}", fg="red")
1269
+ click.echo(_style(f" ❌ {phase.value}", fg="red"))
1196
1270
  elif status == Status.ERROR:
1197
- click.secho(f" 🚫 {phase.value}", fg="red")
1271
+ click.echo(_style(f" 🚫 {phase.value}", fg="red"))
1198
1272
  elif status == Status.INTERRUPTED:
1199
- click.secho(f" ⚡ {phase.value}", fg="yellow")
1273
+ click.echo(_style(f" ⚡ {phase.value}", fg="yellow"))
1200
1274
  click.echo()
1201
1275
 
1202
1276
  def display_test_cases(self, ctx: ExecutionContext) -> None:
1203
1277
  if ctx.statistic.total_cases == 0:
1204
- click.secho("Test cases:", bold=True)
1205
- click.secho(" No test cases were generated\n")
1278
+ click.echo(_style("Test cases:", bold=True))
1279
+ click.echo(" No test cases were generated\n")
1206
1280
  return
1207
1281
 
1208
1282
  unique_failures = sum(
1209
1283
  len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
1210
1284
  )
1211
- click.secho("Test cases:", bold=True)
1285
+ click.echo(_style("Test cases:", bold=True))
1212
1286
 
1213
1287
  parts = [f" {click.style(str(ctx.statistic.total_cases), bold=True)} generated"]
1214
1288
 
@@ -1227,7 +1301,7 @@ class OutputHandler(EventHandler):
1227
1301
  if ctx.statistic.cases_without_checks > 0:
1228
1302
  parts.append(f"{click.style(str(ctx.statistic.cases_without_checks), bold=True)} skipped")
1229
1303
 
1230
- click.secho(", ".join(parts) + "\n")
1304
+ click.echo(_style(", ".join(parts) + "\n"))
1231
1305
 
1232
1306
  def display_failures_summary(self, ctx: ExecutionContext) -> None:
1233
1307
  # Collect all unique failures and their counts by title
@@ -1238,14 +1312,14 @@ class OutputHandler(EventHandler):
1238
1312
  data = failure_counts.get(failure.title, (failure.severity, 0))
1239
1313
  failure_counts[failure.title] = (failure.severity, data[1] + 1)
1240
1314
 
1241
- click.secho("Failures:", bold=True)
1315
+ click.echo(_style("Failures:", bold=True))
1242
1316
 
1243
1317
  # Sort by severity first, then by title
1244
1318
  sorted_failures = sorted(failure_counts.items(), key=lambda x: (x[1][0], x[0]))
1245
1319
 
1246
1320
  for title, (_, count) in sorted_failures:
1247
- click.secho(f" ❌ {title}: ", nl=False)
1248
- click.secho(str(count), bold=True)
1321
+ click.echo(_style(f" ❌ {title}: "), nl=False)
1322
+ click.echo(_style(str(count), bold=True))
1249
1323
  click.echo()
1250
1324
 
1251
1325
  def display_errors_summary(self) -> None:
@@ -1255,11 +1329,11 @@ class OutputHandler(EventHandler):
1255
1329
  title = error.info.title
1256
1330
  error_counts[title] = error_counts.get(title, 0) + 1
1257
1331
 
1258
- click.secho("Errors:", bold=True)
1332
+ click.echo(_style("Errors:", bold=True))
1259
1333
 
1260
1334
  for title in sorted(error_counts):
1261
- click.secho(f" 🚫 {title}: ", nl=False)
1262
- click.secho(str(error_counts[title]), bold=True)
1335
+ click.echo(_style(f" 🚫 {title}: "), nl=False)
1336
+ click.echo(_style(str(error_counts[title]), bold=True))
1263
1337
  click.echo()
1264
1338
 
1265
1339
  def display_final_line(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
@@ -1269,14 +1343,17 @@ class OutputHandler(EventHandler):
1269
1343
  len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
1270
1344
  )
1271
1345
  if unique_failures:
1272
- parts.append(f"{unique_failures} failures")
1346
+ suffix = "s" if unique_failures > 1 else ""
1347
+ parts.append(f"{unique_failures} failure{suffix}")
1273
1348
 
1274
1349
  if self.errors:
1275
- parts.append(f"{len(self.errors)} errors")
1350
+ suffix = "s" if len(self.errors) > 1 else ""
1351
+ parts.append(f"{len(self.errors)} error{suffix}")
1276
1352
 
1277
1353
  total_warnings = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
1278
1354
  if total_warnings:
1279
- parts.append(f"{total_warnings} warnings")
1355
+ suffix = "s" if total_warnings > 1 else ""
1356
+ parts.append(f"{total_warnings} warning{suffix}")
1280
1357
 
1281
1358
  if parts:
1282
1359
  message = f"{', '.join(parts)} in {event.running_time:.2f}s"
@@ -1291,35 +1368,50 @@ class OutputHandler(EventHandler):
1291
1368
  display_section_name(message, fg=color)
1292
1369
 
1293
1370
  def display_reports(self) -> None:
1294
- reports = []
1295
- if self.cassette_config is not None:
1296
- format_name = self.cassette_config.format.name.upper()
1297
- reports.append((format_name, self.cassette_config.path.name))
1298
- if self.junit_xml_file is not None:
1299
- reports.append(("JUnit XML", self.junit_xml_file))
1300
-
1301
- if reports:
1302
- click.secho("Reports:", bold=True)
1371
+ if self.report_config is not None:
1372
+ reports = [
1373
+ (format.value.upper(), self.report_config.get_path(format).name)
1374
+ for format in ReportFormat
1375
+ if format in self.report_config.formats
1376
+ ]
1377
+
1378
+ click.echo(_style("Reports:", bold=True))
1303
1379
  for report_type, path in reports:
1304
- click.secho(f" {report_type}: {path}")
1380
+ click.echo(_style(f" - {report_type}: {path}"))
1305
1381
  click.echo()
1306
1382
 
1383
+ def display_seed(self) -> None:
1384
+ click.echo(_style("Seed: ", bold=True), nl=False)
1385
+ if self.seed is None:
1386
+ click.echo("not used in the deterministic mode")
1387
+ else:
1388
+ click.echo(str(self.seed))
1389
+ click.echo()
1390
+
1307
1391
  def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
1392
+ assert self.loading_manager is None
1393
+ assert self.probing_manager is None
1394
+ assert self.unit_tests_manager is None
1395
+ assert self.stateful_tests_manager is None
1308
1396
  if self.errors:
1309
1397
  display_section_name("ERRORS")
1310
1398
  errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label))
1311
1399
  for error in errors:
1312
1400
  display_section_name(error.label, "_", fg="red")
1313
1401
  click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1314
- click.secho(
1315
- f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
1316
- fg="red",
1402
+ click.echo(
1403
+ _style(
1404
+ f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
1405
+ fg="red",
1406
+ )
1317
1407
  )
1318
1408
  display_failures(ctx)
1319
1409
  if self.warnings.missing_auth:
1320
1410
  self.display_warnings()
1321
1411
  if GLOBAL_EXPERIMENTS.enabled:
1322
1412
  self.display_experiments()
1413
+ if ctx.statistic.extraction_failures:
1414
+ self.display_stateful_failures(ctx)
1323
1415
  display_section_name("SUMMARY")
1324
1416
  click.echo()
1325
1417
 
@@ -1336,8 +1428,8 @@ class OutputHandler(EventHandler):
1336
1428
 
1337
1429
  if self.warnings.missing_auth:
1338
1430
  affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
1339
- click.secho("Warnings:", bold=True)
1340
- click.secho(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow")
1431
+ click.echo(_style("Warnings:", bold=True))
1432
+ click.echo(_style(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
1341
1433
  click.echo()
1342
1434
 
1343
1435
  if ctx.summary_lines:
@@ -1346,6 +1438,7 @@ class OutputHandler(EventHandler):
1346
1438
 
1347
1439
  self.display_test_cases(ctx)
1348
1440
  self.display_reports()
1441
+ self.display_seed()
1349
1442
  self.display_final_line(ctx, event)
1350
1443
 
1351
1444