schemathesis 4.0.0a2__py3-none-any.whl → 4.0.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/cli/__init__.py +15 -4
- schemathesis/cli/commands/run/__init__.py +148 -94
- schemathesis/cli/commands/run/context.py +72 -2
- schemathesis/cli/commands/run/events.py +22 -2
- schemathesis/cli/commands/run/executor.py +35 -12
- schemathesis/cli/commands/run/filters.py +1 -0
- schemathesis/cli/commands/run/handlers/cassettes.py +27 -46
- schemathesis/cli/commands/run/handlers/junitxml.py +1 -1
- schemathesis/cli/commands/run/handlers/output.py +180 -87
- schemathesis/cli/commands/run/hypothesis.py +30 -19
- schemathesis/cli/commands/run/reports.py +72 -0
- schemathesis/cli/commands/run/validation.py +18 -12
- schemathesis/cli/ext/groups.py +42 -13
- schemathesis/cli/ext/options.py +15 -8
- schemathesis/core/errors.py +85 -9
- schemathesis/core/failures.py +2 -1
- schemathesis/core/transforms.py +1 -1
- schemathesis/engine/core.py +1 -1
- schemathesis/engine/errors.py +17 -6
- schemathesis/engine/phases/stateful/__init__.py +1 -0
- schemathesis/engine/phases/stateful/_executor.py +9 -12
- schemathesis/engine/phases/unit/__init__.py +2 -3
- schemathesis/engine/phases/unit/_executor.py +16 -13
- schemathesis/engine/recorder.py +22 -21
- schemathesis/errors.py +23 -13
- schemathesis/filters.py +8 -0
- schemathesis/generation/coverage.py +10 -5
- schemathesis/generation/hypothesis/builder.py +15 -12
- schemathesis/generation/stateful/state_machine.py +57 -12
- schemathesis/pytest/lazy.py +2 -3
- schemathesis/pytest/plugin.py +2 -3
- schemathesis/schemas.py +1 -1
- schemathesis/specs/openapi/checks.py +77 -37
- schemathesis/specs/openapi/expressions/__init__.py +22 -6
- schemathesis/specs/openapi/expressions/nodes.py +15 -21
- schemathesis/specs/openapi/expressions/parser.py +1 -1
- schemathesis/specs/openapi/parameters.py +0 -2
- schemathesis/specs/openapi/patterns.py +170 -2
- schemathesis/specs/openapi/schemas.py +67 -39
- schemathesis/specs/openapi/stateful/__init__.py +207 -84
- schemathesis/specs/openapi/stateful/control.py +87 -0
- schemathesis/specs/openapi/{links.py → stateful/links.py} +72 -14
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/METADATA +1 -1
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/RECORD +47 -45
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a4.dist-info}/licenses/LICENSE +0 -0
@@ -1,8 +1,10 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
import os
|
4
|
+
import textwrap
|
4
5
|
import time
|
5
6
|
from dataclasses import dataclass, field
|
7
|
+
from json.decoder import JSONDecodeError
|
6
8
|
from types import GeneratorType
|
7
9
|
from typing import TYPE_CHECKING, Any, Generator, Iterable
|
8
10
|
|
@@ -11,11 +13,12 @@ import click
|
|
11
13
|
from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
|
12
14
|
from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
|
13
15
|
from schemathesis.cli.commands.run.handlers.base import EventHandler
|
14
|
-
from schemathesis.cli.commands.run.
|
16
|
+
from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
|
15
17
|
from schemathesis.cli.constants import ISSUE_TRACKER_URL
|
16
18
|
from schemathesis.cli.core import get_terminal_width
|
17
19
|
from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
|
18
20
|
from schemathesis.core.failures import MessageBlock, Severity, format_failures
|
21
|
+
from schemathesis.core.output import prepare_response_payload
|
19
22
|
from schemathesis.core.result import Err, Ok
|
20
23
|
from schemathesis.core.version import SCHEMATHESIS_VERSION
|
21
24
|
from schemathesis.engine import Status, events
|
@@ -32,6 +35,8 @@ if TYPE_CHECKING:
|
|
32
35
|
from rich.progress import Progress, TaskID
|
33
36
|
from rich.text import Text
|
34
37
|
|
38
|
+
from schemathesis.generation.stateful.state_machine import ExtractionFailure
|
39
|
+
|
35
40
|
IO_ENCODING = os.getenv("PYTHONIOENCODING", "utf-8")
|
36
41
|
DISCORD_LINK = "https://discord.gg/R9ASRAmHnA"
|
37
42
|
|
@@ -40,7 +45,7 @@ def display_section_name(title: str, separator: str = "=", **kwargs: Any) -> Non
|
|
40
45
|
"""Print section name with separators in terminal with the given title nicely centered."""
|
41
46
|
message = f" {title} ".center(get_terminal_width(), separator)
|
42
47
|
kwargs.setdefault("bold", True)
|
43
|
-
click.
|
48
|
+
click.echo(_style(message, **kwargs))
|
44
49
|
|
45
50
|
|
46
51
|
def bold(option: str) -> str:
|
@@ -58,12 +63,14 @@ def display_failures(ctx: ExecutionContext) -> None:
|
|
58
63
|
|
59
64
|
|
60
65
|
if IO_ENCODING != "utf-8":
|
66
|
+
HEADER_SEPARATOR = "-"
|
61
67
|
|
62
68
|
def _style(text: str, **kwargs: Any) -> str:
|
63
69
|
text = text.encode(IO_ENCODING, errors="replace").decode("utf-8")
|
64
70
|
return click.style(text, **kwargs)
|
65
71
|
|
66
72
|
else:
|
73
|
+
HEADER_SEPARATOR = "━"
|
67
74
|
|
68
75
|
def _style(text: str, **kwargs: Any) -> str:
|
69
76
|
return click.style(text, **kwargs)
|
@@ -98,7 +105,7 @@ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks:
|
|
98
105
|
|
99
106
|
|
100
107
|
VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema or GraphQL endpoint"
|
101
|
-
DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--
|
108
|
+
DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--tls-verify=false`')}."
|
102
109
|
LOADER_ERROR_SUGGESTIONS = {
|
103
110
|
# SSL-specific connection issue
|
104
111
|
LoaderErrorKind.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
|
@@ -122,14 +129,14 @@ def _display_extras(extras: list[str]) -> None:
|
|
122
129
|
if extras:
|
123
130
|
click.echo()
|
124
131
|
for extra in extras:
|
125
|
-
click.
|
132
|
+
click.echo(_style(f" {extra}"))
|
126
133
|
|
127
134
|
|
128
135
|
def display_header(version: str) -> None:
|
129
136
|
prefix = "v" if version != "dev" else ""
|
130
137
|
header = f"Schemathesis {prefix}{version}"
|
131
|
-
click.
|
132
|
-
click.
|
138
|
+
click.echo(_style(header, bold=True))
|
139
|
+
click.echo(_style(HEADER_SEPARATOR * len(header), bold=True))
|
133
140
|
click.echo()
|
134
141
|
|
135
142
|
|
@@ -377,6 +384,7 @@ class UnitTestProgressManager:
|
|
377
384
|
|
378
385
|
def __init__(
|
379
386
|
self,
|
387
|
+
*,
|
380
388
|
console: Console,
|
381
389
|
title: str,
|
382
390
|
total: int,
|
@@ -435,6 +443,7 @@ class UnitTestProgressManager:
|
|
435
443
|
Status.FAILURE: 0,
|
436
444
|
Status.SKIP: 0,
|
437
445
|
Status.ERROR: 0,
|
446
|
+
Status.INTERRUPTED: 0,
|
438
447
|
}
|
439
448
|
self._update_stats_display()
|
440
449
|
|
@@ -451,8 +460,8 @@ class UnitTestProgressManager:
|
|
451
460
|
parts.append(f"❌ {self.stats[Status.FAILURE]:{width}d} failed")
|
452
461
|
if self.stats[Status.ERROR]:
|
453
462
|
parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} errors")
|
454
|
-
if self.stats[Status.SKIP]:
|
455
|
-
parts.append(f"⏭️ {self.stats[Status.SKIP]:{width}d} skipped")
|
463
|
+
if self.stats[Status.SKIP] or self.stats[Status.INTERRUPTED]:
|
464
|
+
parts.append(f"⏭️ {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
|
456
465
|
return " ".join(parts)
|
457
466
|
|
458
467
|
def _update_stats_display(self) -> None:
|
@@ -558,6 +567,7 @@ class StatefulProgressManager:
|
|
558
567
|
|
559
568
|
console: Console
|
560
569
|
title: str
|
570
|
+
links_selected: int
|
561
571
|
links_total: int
|
562
572
|
start_time: float
|
563
573
|
|
@@ -574,13 +584,14 @@ class StatefulProgressManager:
|
|
574
584
|
|
575
585
|
# State
|
576
586
|
scenarios: int
|
577
|
-
|
587
|
+
links_covered: set[str]
|
578
588
|
stats: dict[Status, int]
|
579
589
|
is_interrupted: bool
|
580
590
|
|
581
591
|
__slots__ = (
|
582
592
|
"console",
|
583
593
|
"title",
|
594
|
+
"links_selected",
|
584
595
|
"links_total",
|
585
596
|
"start_time",
|
586
597
|
"title_progress",
|
@@ -591,17 +602,18 @@ class StatefulProgressManager:
|
|
591
602
|
"progress_task_id",
|
592
603
|
"stats_task_id",
|
593
604
|
"scenarios",
|
594
|
-
"
|
605
|
+
"links_covered",
|
595
606
|
"stats",
|
596
607
|
"is_interrupted",
|
597
608
|
)
|
598
609
|
|
599
|
-
def __init__(self, console: Console, title: str, links_total: int) -> None:
|
610
|
+
def __init__(self, *, console: Console, title: str, links_selected: int, links_total: int) -> None:
|
600
611
|
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
601
612
|
from rich.style import Style
|
602
613
|
|
603
614
|
self.console = console
|
604
615
|
self.title = title
|
616
|
+
self.links_selected = links_selected
|
605
617
|
self.links_total = links_total
|
606
618
|
self.start_time = time.monotonic()
|
607
619
|
|
@@ -633,7 +645,7 @@ class StatefulProgressManager:
|
|
633
645
|
|
634
646
|
# Initialize state
|
635
647
|
self.scenarios = 0
|
636
|
-
self.
|
648
|
+
self.links_covered = set()
|
637
649
|
self.stats = {
|
638
650
|
Status.SUCCESS: 0,
|
639
651
|
Status.FAILURE: 0,
|
@@ -650,7 +662,9 @@ class StatefulProgressManager:
|
|
650
662
|
|
651
663
|
# Initialize progress displays
|
652
664
|
self.title_task_id = self.title_progress.add_task("Stateful tests")
|
653
|
-
self.progress_task_id = self.progress_bar.add_task(
|
665
|
+
self.progress_task_id = self.progress_bar.add_task(
|
666
|
+
"", scenarios=0, links=f"0 covered / {self.links_selected} selected / {self.links_total} total links"
|
667
|
+
)
|
654
668
|
|
655
669
|
# Create live display
|
656
670
|
group = Group(
|
@@ -668,10 +682,10 @@ class StatefulProgressManager:
|
|
668
682
|
if self.live:
|
669
683
|
self.live.stop()
|
670
684
|
|
671
|
-
def update(self,
|
685
|
+
def update(self, links_covered: set[str], status: Status | None = None) -> None:
|
672
686
|
"""Update progress and stats."""
|
673
687
|
self.scenarios += 1
|
674
|
-
self.
|
688
|
+
self.links_covered.update(links_covered)
|
675
689
|
|
676
690
|
if status is not None:
|
677
691
|
self.stats[status] += 1
|
@@ -685,7 +699,7 @@ class StatefulProgressManager:
|
|
685
699
|
self.progress_bar.update(
|
686
700
|
self.progress_task_id,
|
687
701
|
scenarios=self.scenarios,
|
688
|
-
links=f"{len(self.
|
702
|
+
links=f"{len(self.links_covered)} covered / {self.links_selected} selected / {self.links_total} total links",
|
689
703
|
)
|
690
704
|
|
691
705
|
def _get_stats_message(self) -> str:
|
@@ -770,6 +784,8 @@ def format_duration(duration_ms: int) -> str:
|
|
770
784
|
@dataclass
|
771
785
|
class OutputHandler(EventHandler):
|
772
786
|
workers_num: int
|
787
|
+
# Seed can't be absent in the deterministic mode
|
788
|
+
seed: int | None
|
773
789
|
rate_limit: str | None
|
774
790
|
wait_for_schema: float | None
|
775
791
|
|
@@ -780,8 +796,7 @@ class OutputHandler(EventHandler):
|
|
780
796
|
|
781
797
|
statistic: ApiStatistic | None = None
|
782
798
|
skip_reasons: list[str] = field(default_factory=list)
|
783
|
-
|
784
|
-
junit_xml_file: str | None = None
|
799
|
+
report_config: ReportConfig | None = None
|
785
800
|
warnings: WarningData = field(default_factory=WarningData)
|
786
801
|
errors: list[events.NonFatalError] = field(default_factory=list)
|
787
802
|
phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
|
@@ -856,7 +871,8 @@ class OutputHandler(EventHandler):
|
|
856
871
|
|
857
872
|
table.add_row("Base URL:", event.base_url)
|
858
873
|
table.add_row("Specification:", event.specification.name)
|
859
|
-
|
874
|
+
statistic = event.statistic.operations
|
875
|
+
table.add_row("Operations:", f"{statistic.selected} selected / {statistic.total} total")
|
860
876
|
|
861
877
|
message = Padding(table, BLOCK_PADDING)
|
862
878
|
self.console.print(message)
|
@@ -883,7 +899,7 @@ class OutputHandler(EventHandler):
|
|
883
899
|
self.unit_tests_manager = UnitTestProgressManager(
|
884
900
|
console=self.console,
|
885
901
|
title="Unit tests",
|
886
|
-
total=self.statistic.operations.
|
902
|
+
total=self.statistic.operations.selected,
|
887
903
|
)
|
888
904
|
self.unit_tests_manager.start()
|
889
905
|
|
@@ -892,6 +908,7 @@ class OutputHandler(EventHandler):
|
|
892
908
|
self.stateful_tests_manager = StatefulProgressManager(
|
893
909
|
console=self.console,
|
894
910
|
title="Stateful tests",
|
911
|
+
links_selected=self.statistic.links.selected,
|
895
912
|
links_total=self.statistic.links.total,
|
896
913
|
)
|
897
914
|
self.stateful_tests_manager.start()
|
@@ -984,7 +1001,8 @@ class OutputHandler(EventHandler):
|
|
984
1001
|
table.add_column("Value", style="cyan")
|
985
1002
|
table.add_row("Scenarios:", f"{self.stateful_tests_manager.scenarios}")
|
986
1003
|
table.add_row(
|
987
|
-
"API Links:",
|
1004
|
+
"API Links:",
|
1005
|
+
f"{len(self.stateful_tests_manager.links_covered)} covered / {self.stateful_tests_manager.links_selected} selected / {self.stateful_tests_manager.links_total} total",
|
988
1006
|
)
|
989
1007
|
|
990
1008
|
self.console.print()
|
@@ -1081,7 +1099,7 @@ class OutputHandler(EventHandler):
|
|
1081
1099
|
if not (event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.wait_for_schema is not None):
|
1082
1100
|
suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
|
1083
1101
|
if suggestion is not None:
|
1084
|
-
click.
|
1102
|
+
click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
1085
1103
|
|
1086
1104
|
raise click.Abort
|
1087
1105
|
title = "Test Execution Error"
|
@@ -1089,16 +1107,16 @@ class OutputHandler(EventHandler):
|
|
1089
1107
|
traceback = format_exception(event.exception, with_traceback=True)
|
1090
1108
|
extras = split_traceback(traceback)
|
1091
1109
|
suggestion = f"Please consider reporting the traceback above to our issue tracker:\n\n {ISSUE_TRACKER_URL}."
|
1092
|
-
click.
|
1110
|
+
click.echo(_style(title, fg="red", bold=True))
|
1093
1111
|
click.echo()
|
1094
|
-
click.
|
1112
|
+
click.echo(message)
|
1095
1113
|
_display_extras(extras)
|
1096
1114
|
if not (
|
1097
1115
|
isinstance(event.exception, LoaderError)
|
1098
1116
|
and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
|
1099
1117
|
and self.wait_for_schema is not None
|
1100
1118
|
):
|
1101
|
-
click.
|
1119
|
+
click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
1102
1120
|
|
1103
1121
|
raise click.Abort
|
1104
1122
|
|
@@ -1106,29 +1124,33 @@ class OutputHandler(EventHandler):
|
|
1106
1124
|
display_section_name("WARNINGS")
|
1107
1125
|
total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
|
1108
1126
|
suffix = "" if total == 1 else "s"
|
1109
|
-
click.
|
1110
|
-
|
1111
|
-
|
1127
|
+
click.echo(
|
1128
|
+
_style(
|
1129
|
+
f"\nMissing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
|
1130
|
+
fg="yellow",
|
1131
|
+
)
|
1112
1132
|
)
|
1113
1133
|
|
1114
1134
|
for status_code, operations in self.warnings.missing_auth.items():
|
1115
1135
|
status_text = "Unauthorized" if status_code == 401 else "Forbidden"
|
1116
1136
|
count = len(operations)
|
1117
1137
|
suffix = "" if count == 1 else "s"
|
1118
|
-
click.
|
1119
|
-
|
1120
|
-
|
1138
|
+
click.echo(
|
1139
|
+
_style(
|
1140
|
+
f"{status_code} {status_text} ({count} operation{suffix}):",
|
1141
|
+
fg="yellow",
|
1142
|
+
)
|
1121
1143
|
)
|
1122
1144
|
# Show first few API operations
|
1123
1145
|
for endpoint in operations[:3]:
|
1124
|
-
click.
|
1146
|
+
click.echo(_style(f" - {endpoint}", fg="yellow"))
|
1125
1147
|
if len(operations) > 3:
|
1126
|
-
click.
|
1148
|
+
click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
|
1127
1149
|
click.echo()
|
1128
|
-
click.
|
1129
|
-
click.
|
1130
|
-
click.
|
1131
|
-
click.
|
1150
|
+
click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
|
1151
|
+
click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
|
1152
|
+
click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
|
1153
|
+
click.echo(_style("to provide authentication credentials", fg="yellow"))
|
1132
1154
|
click.echo()
|
1133
1155
|
|
1134
1156
|
def display_experiments(self) -> None:
|
@@ -1136,26 +1158,78 @@ class OutputHandler(EventHandler):
|
|
1136
1158
|
|
1137
1159
|
click.echo()
|
1138
1160
|
for experiment in sorted(GLOBAL_EXPERIMENTS.enabled, key=lambda e: e.name):
|
1139
|
-
click.
|
1140
|
-
click.
|
1141
|
-
click.
|
1161
|
+
click.echo(_style(f"🧪 {experiment.name}: ", bold=True), nl=False)
|
1162
|
+
click.echo(_style(experiment.description))
|
1163
|
+
click.echo(_style(f" Feedback: {experiment.discussion_url}"))
|
1142
1164
|
click.echo()
|
1143
1165
|
|
1144
|
-
click.
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1166
|
+
click.echo(
|
1167
|
+
_style(
|
1168
|
+
"Your feedback is crucial for experimental features. "
|
1169
|
+
"Please visit the provided URL(s) to share your thoughts.",
|
1170
|
+
dim=True,
|
1171
|
+
)
|
1148
1172
|
)
|
1149
1173
|
click.echo()
|
1150
1174
|
|
1175
|
+
def display_stateful_failures(self, ctx: ExecutionContext) -> None:
|
1176
|
+
display_section_name("Stateful tests")
|
1177
|
+
|
1178
|
+
click.echo("\nFailed to extract data from response:")
|
1179
|
+
|
1180
|
+
grouped: dict[str, list[ExtractionFailure]] = {}
|
1181
|
+
for failure in ctx.statistic.extraction_failures:
|
1182
|
+
grouped.setdefault(failure.id, []).append(failure)
|
1183
|
+
|
1184
|
+
for idx, (transition_id, failures) in enumerate(grouped.items(), 1):
|
1185
|
+
for failure in failures:
|
1186
|
+
click.echo(f"\n {idx}. Test Case ID: {failure.case_id}\n")
|
1187
|
+
click.echo(f" {transition_id}")
|
1188
|
+
|
1189
|
+
indent = " "
|
1190
|
+
if failure.error:
|
1191
|
+
if isinstance(failure.error, JSONDecodeError):
|
1192
|
+
click.echo(f"\n{indent}Failed to parse JSON from response")
|
1193
|
+
else:
|
1194
|
+
click.echo(f"\n{indent}{failure.error.__class__.__name__}: {failure.error}")
|
1195
|
+
else:
|
1196
|
+
description = (
|
1197
|
+
f"\n{indent}Could not resolve parameter `{failure.parameter_name}` via `{failure.expression}`"
|
1198
|
+
)
|
1199
|
+
prefix = "$response.body"
|
1200
|
+
if failure.expression.startswith(prefix):
|
1201
|
+
description += f"\n{indent}Path `{failure.expression[len(prefix) :]}` not found in response"
|
1202
|
+
click.echo(description)
|
1203
|
+
|
1204
|
+
click.echo()
|
1205
|
+
|
1206
|
+
for case, response in reversed(failure.history):
|
1207
|
+
curl = case.as_curl_command(headers=dict(response.request.headers), verify=response.verify)
|
1208
|
+
click.echo(f"{indent}[{response.status_code}] {curl}")
|
1209
|
+
|
1210
|
+
response = failure.response
|
1211
|
+
|
1212
|
+
if response.content is None or not response.content:
|
1213
|
+
click.echo(f"\n{indent}<EMPTY>")
|
1214
|
+
else:
|
1215
|
+
try:
|
1216
|
+
payload = prepare_response_payload(response.text, config=ctx.output_config)
|
1217
|
+
click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
|
1218
|
+
except UnicodeDecodeError:
|
1219
|
+
click.echo(f"\n{indent}<BINARY>")
|
1220
|
+
|
1221
|
+
click.echo()
|
1222
|
+
|
1151
1223
|
def display_api_operations(self, ctx: ExecutionContext) -> None:
|
1152
1224
|
assert self.statistic is not None
|
1153
|
-
click.
|
1154
|
-
click.
|
1155
|
-
|
1156
|
-
|
1225
|
+
click.echo(_style("API Operations:", bold=True))
|
1226
|
+
click.echo(
|
1227
|
+
_style(
|
1228
|
+
f" Selected: {click.style(str(self.statistic.operations.selected), bold=True)}/"
|
1229
|
+
f"{click.style(str(self.statistic.operations.total), bold=True)}"
|
1230
|
+
)
|
1157
1231
|
)
|
1158
|
-
click.
|
1232
|
+
click.echo(_style(f" Tested: {click.style(str(len(ctx.statistic.tested_operations)), bold=True)}"))
|
1159
1233
|
errors = len(
|
1160
1234
|
{
|
1161
1235
|
err.label
|
@@ -1167,48 +1241,48 @@ class OutputHandler(EventHandler):
|
|
1167
1241
|
}
|
1168
1242
|
)
|
1169
1243
|
if errors:
|
1170
|
-
click.
|
1244
|
+
click.echo(_style(f" Errored: {click.style(str(errors), bold=True)}"))
|
1171
1245
|
|
1172
1246
|
# API operations that are skipped due to fail-fast are counted here as well
|
1173
1247
|
total_skips = self.statistic.operations.selected - len(ctx.statistic.tested_operations) - errors
|
1174
1248
|
if total_skips:
|
1175
|
-
click.
|
1249
|
+
click.echo(_style(f" Skipped: {click.style(str(total_skips), bold=True)}"))
|
1176
1250
|
for reason in sorted(set(self.skip_reasons)):
|
1177
|
-
click.
|
1251
|
+
click.echo(_style(f" - {reason.rstrip('.')}"))
|
1178
1252
|
click.echo()
|
1179
1253
|
|
1180
1254
|
def display_phases(self) -> None:
|
1181
|
-
click.
|
1255
|
+
click.echo(_style("Test Phases:", bold=True))
|
1182
1256
|
|
1183
1257
|
for phase in PhaseName:
|
1184
1258
|
status, skip_reason = self.phases[phase]
|
1185
1259
|
|
1186
1260
|
if status == Status.SKIP:
|
1187
|
-
click.
|
1261
|
+
click.echo(_style(f" ⏭️ {phase.value}", fg="yellow"), nl=False)
|
1188
1262
|
if skip_reason:
|
1189
|
-
click.
|
1263
|
+
click.echo(_style(f" ({skip_reason.value})", fg="yellow"))
|
1190
1264
|
else:
|
1191
1265
|
click.echo()
|
1192
1266
|
elif status == Status.SUCCESS:
|
1193
|
-
click.
|
1267
|
+
click.echo(_style(f" ✅ {phase.value}", fg="green"))
|
1194
1268
|
elif status == Status.FAILURE:
|
1195
|
-
click.
|
1269
|
+
click.echo(_style(f" ❌ {phase.value}", fg="red"))
|
1196
1270
|
elif status == Status.ERROR:
|
1197
|
-
click.
|
1271
|
+
click.echo(_style(f" 🚫 {phase.value}", fg="red"))
|
1198
1272
|
elif status == Status.INTERRUPTED:
|
1199
|
-
click.
|
1273
|
+
click.echo(_style(f" ⚡ {phase.value}", fg="yellow"))
|
1200
1274
|
click.echo()
|
1201
1275
|
|
1202
1276
|
def display_test_cases(self, ctx: ExecutionContext) -> None:
|
1203
1277
|
if ctx.statistic.total_cases == 0:
|
1204
|
-
click.
|
1205
|
-
click.
|
1278
|
+
click.echo(_style("Test cases:", bold=True))
|
1279
|
+
click.echo(" No test cases were generated\n")
|
1206
1280
|
return
|
1207
1281
|
|
1208
1282
|
unique_failures = sum(
|
1209
1283
|
len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
|
1210
1284
|
)
|
1211
|
-
click.
|
1285
|
+
click.echo(_style("Test cases:", bold=True))
|
1212
1286
|
|
1213
1287
|
parts = [f" {click.style(str(ctx.statistic.total_cases), bold=True)} generated"]
|
1214
1288
|
|
@@ -1227,7 +1301,7 @@ class OutputHandler(EventHandler):
|
|
1227
1301
|
if ctx.statistic.cases_without_checks > 0:
|
1228
1302
|
parts.append(f"{click.style(str(ctx.statistic.cases_without_checks), bold=True)} skipped")
|
1229
1303
|
|
1230
|
-
click.
|
1304
|
+
click.echo(_style(", ".join(parts) + "\n"))
|
1231
1305
|
|
1232
1306
|
def display_failures_summary(self, ctx: ExecutionContext) -> None:
|
1233
1307
|
# Collect all unique failures and their counts by title
|
@@ -1238,14 +1312,14 @@ class OutputHandler(EventHandler):
|
|
1238
1312
|
data = failure_counts.get(failure.title, (failure.severity, 0))
|
1239
1313
|
failure_counts[failure.title] = (failure.severity, data[1] + 1)
|
1240
1314
|
|
1241
|
-
click.
|
1315
|
+
click.echo(_style("Failures:", bold=True))
|
1242
1316
|
|
1243
1317
|
# Sort by severity first, then by title
|
1244
1318
|
sorted_failures = sorted(failure_counts.items(), key=lambda x: (x[1][0], x[0]))
|
1245
1319
|
|
1246
1320
|
for title, (_, count) in sorted_failures:
|
1247
|
-
click.
|
1248
|
-
click.
|
1321
|
+
click.echo(_style(f" ❌ {title}: "), nl=False)
|
1322
|
+
click.echo(_style(str(count), bold=True))
|
1249
1323
|
click.echo()
|
1250
1324
|
|
1251
1325
|
def display_errors_summary(self) -> None:
|
@@ -1255,11 +1329,11 @@ class OutputHandler(EventHandler):
|
|
1255
1329
|
title = error.info.title
|
1256
1330
|
error_counts[title] = error_counts.get(title, 0) + 1
|
1257
1331
|
|
1258
|
-
click.
|
1332
|
+
click.echo(_style("Errors:", bold=True))
|
1259
1333
|
|
1260
1334
|
for title in sorted(error_counts):
|
1261
|
-
click.
|
1262
|
-
click.
|
1335
|
+
click.echo(_style(f" 🚫 {title}: "), nl=False)
|
1336
|
+
click.echo(_style(str(error_counts[title]), bold=True))
|
1263
1337
|
click.echo()
|
1264
1338
|
|
1265
1339
|
def display_final_line(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
|
@@ -1269,14 +1343,17 @@ class OutputHandler(EventHandler):
|
|
1269
1343
|
len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
|
1270
1344
|
)
|
1271
1345
|
if unique_failures:
|
1272
|
-
|
1346
|
+
suffix = "s" if unique_failures > 1 else ""
|
1347
|
+
parts.append(f"{unique_failures} failure{suffix}")
|
1273
1348
|
|
1274
1349
|
if self.errors:
|
1275
|
-
|
1350
|
+
suffix = "s" if len(self.errors) > 1 else ""
|
1351
|
+
parts.append(f"{len(self.errors)} error{suffix}")
|
1276
1352
|
|
1277
1353
|
total_warnings = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
|
1278
1354
|
if total_warnings:
|
1279
|
-
|
1355
|
+
suffix = "s" if total_warnings > 1 else ""
|
1356
|
+
parts.append(f"{total_warnings} warning{suffix}")
|
1280
1357
|
|
1281
1358
|
if parts:
|
1282
1359
|
message = f"{', '.join(parts)} in {event.running_time:.2f}s"
|
@@ -1291,35 +1368,50 @@ class OutputHandler(EventHandler):
|
|
1291
1368
|
display_section_name(message, fg=color)
|
1292
1369
|
|
1293
1370
|
def display_reports(self) -> None:
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1301
|
-
|
1302
|
-
click.secho("Reports:", bold=True)
|
1371
|
+
if self.report_config is not None:
|
1372
|
+
reports = [
|
1373
|
+
(format.value.upper(), self.report_config.get_path(format).name)
|
1374
|
+
for format in ReportFormat
|
1375
|
+
if format in self.report_config.formats
|
1376
|
+
]
|
1377
|
+
|
1378
|
+
click.echo(_style("Reports:", bold=True))
|
1303
1379
|
for report_type, path in reports:
|
1304
|
-
click.
|
1380
|
+
click.echo(_style(f" - {report_type}: {path}"))
|
1305
1381
|
click.echo()
|
1306
1382
|
|
1383
|
+
def display_seed(self) -> None:
|
1384
|
+
click.echo(_style("Seed: ", bold=True), nl=False)
|
1385
|
+
if self.seed is None:
|
1386
|
+
click.echo("not used in the deterministic mode")
|
1387
|
+
else:
|
1388
|
+
click.echo(str(self.seed))
|
1389
|
+
click.echo()
|
1390
|
+
|
1307
1391
|
def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
|
1392
|
+
assert self.loading_manager is None
|
1393
|
+
assert self.probing_manager is None
|
1394
|
+
assert self.unit_tests_manager is None
|
1395
|
+
assert self.stateful_tests_manager is None
|
1308
1396
|
if self.errors:
|
1309
1397
|
display_section_name("ERRORS")
|
1310
1398
|
errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label))
|
1311
1399
|
for error in errors:
|
1312
1400
|
display_section_name(error.label, "_", fg="red")
|
1313
1401
|
click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
|
1314
|
-
click.
|
1315
|
-
|
1316
|
-
|
1402
|
+
click.echo(
|
1403
|
+
_style(
|
1404
|
+
f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
|
1405
|
+
fg="red",
|
1406
|
+
)
|
1317
1407
|
)
|
1318
1408
|
display_failures(ctx)
|
1319
1409
|
if self.warnings.missing_auth:
|
1320
1410
|
self.display_warnings()
|
1321
1411
|
if GLOBAL_EXPERIMENTS.enabled:
|
1322
1412
|
self.display_experiments()
|
1413
|
+
if ctx.statistic.extraction_failures:
|
1414
|
+
self.display_stateful_failures(ctx)
|
1323
1415
|
display_section_name("SUMMARY")
|
1324
1416
|
click.echo()
|
1325
1417
|
|
@@ -1336,8 +1428,8 @@ class OutputHandler(EventHandler):
|
|
1336
1428
|
|
1337
1429
|
if self.warnings.missing_auth:
|
1338
1430
|
affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
|
1339
|
-
click.
|
1340
|
-
click.
|
1431
|
+
click.echo(_style("Warnings:", bold=True))
|
1432
|
+
click.echo(_style(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
|
1341
1433
|
click.echo()
|
1342
1434
|
|
1343
1435
|
if ctx.summary_lines:
|
@@ -1346,6 +1438,7 @@ class OutputHandler(EventHandler):
|
|
1346
1438
|
|
1347
1439
|
self.display_test_cases(ctx)
|
1348
1440
|
self.display_reports()
|
1441
|
+
self.display_seed()
|
1349
1442
|
self.display_final_line(ctx, event)
|
1350
1443
|
|
1351
1444
|
|