schemathesis 4.0.0a2__py3-none-any.whl → 4.0.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/cli/__init__.py +12 -1
- schemathesis/cli/commands/run/events.py +22 -2
- schemathesis/cli/commands/run/executor.py +3 -0
- schemathesis/cli/commands/run/handlers/output.py +108 -71
- schemathesis/core/errors.py +8 -0
- schemathesis/engine/core.py +1 -1
- schemathesis/engine/errors.py +11 -5
- schemathesis/engine/phases/stateful/__init__.py +1 -0
- schemathesis/engine/phases/stateful/_executor.py +8 -11
- schemathesis/engine/recorder.py +22 -21
- schemathesis/errors.py +19 -13
- schemathesis/generation/coverage.py +4 -4
- schemathesis/generation/hypothesis/builder.py +15 -12
- schemathesis/generation/stateful/state_machine.py +8 -9
- schemathesis/specs/openapi/checks.py +50 -27
- schemathesis/specs/openapi/links.py +18 -4
- schemathesis/specs/openapi/patterns.py +170 -2
- schemathesis/specs/openapi/schemas.py +54 -26
- schemathesis/specs/openapi/stateful/__init__.py +124 -74
- schemathesis/specs/openapi/stateful/control.py +87 -0
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a3.dist-info}/METADATA +1 -1
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a3.dist-info}/RECORD +25 -24
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a3.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a3.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a2.dist-info → schemathesis-4.0.0a3.dist-info}/licenses/LICENSE +0 -0
schemathesis/cli/__init__.py
CHANGED
@@ -1,11 +1,22 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
from schemathesis.cli.commands import Group, run, schemathesis
|
4
|
+
from schemathesis.cli.commands.run.context import ExecutionContext
|
5
|
+
from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
|
4
6
|
from schemathesis.cli.commands.run.executor import handler
|
5
7
|
from schemathesis.cli.commands.run.handlers import EventHandler
|
6
8
|
from schemathesis.cli.ext.groups import GROUPS
|
7
9
|
|
8
|
-
__all__ = [
|
10
|
+
__all__ = [
|
11
|
+
"schemathesis",
|
12
|
+
"run",
|
13
|
+
"EventHandler",
|
14
|
+
"ExecutionContext",
|
15
|
+
"LoadingStarted",
|
16
|
+
"LoadingFinished",
|
17
|
+
"add_group",
|
18
|
+
"handler",
|
19
|
+
]
|
9
20
|
|
10
21
|
|
11
22
|
def add_group(name: str, *, index: int | None = None) -> Group:
|
@@ -16,10 +16,28 @@ class LoadingStarted(events.EngineEvent):
|
|
16
16
|
|
17
17
|
|
18
18
|
class LoadingFinished(events.EngineEvent):
|
19
|
-
__slots__ = (
|
19
|
+
__slots__ = (
|
20
|
+
"id",
|
21
|
+
"timestamp",
|
22
|
+
"location",
|
23
|
+
"duration",
|
24
|
+
"base_url",
|
25
|
+
"base_path",
|
26
|
+
"specification",
|
27
|
+
"statistic",
|
28
|
+
"schema",
|
29
|
+
)
|
20
30
|
|
21
31
|
def __init__(
|
22
|
-
self,
|
32
|
+
self,
|
33
|
+
*,
|
34
|
+
location: str,
|
35
|
+
start_time: float,
|
36
|
+
base_url: str,
|
37
|
+
base_path: str,
|
38
|
+
specification: Specification,
|
39
|
+
statistic: ApiStatistic,
|
40
|
+
schema: dict,
|
23
41
|
) -> None:
|
24
42
|
self.id = uuid.uuid4()
|
25
43
|
self.timestamp = time.time()
|
@@ -28,3 +46,5 @@ class LoadingFinished(events.EngineEvent):
|
|
28
46
|
self.base_url = base_url
|
29
47
|
self.specification = specification
|
30
48
|
self.statistic = statistic
|
49
|
+
self.schema = schema
|
50
|
+
self.base_path = base_path
|
@@ -83,6 +83,8 @@ def into_event_stream(config: RunConfig) -> EventGenerator:
|
|
83
83
|
base_url=schema.get_base_url(),
|
84
84
|
specification=schema.specification,
|
85
85
|
statistic=schema.statistic,
|
86
|
+
schema=schema.raw_schema,
|
87
|
+
base_path=schema.base_path,
|
86
88
|
)
|
87
89
|
|
88
90
|
try:
|
@@ -104,6 +106,7 @@ def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
|
|
104
106
|
handlers.append(
|
105
107
|
OutputHandler(
|
106
108
|
workers_num=config.engine.execution.workers_num,
|
109
|
+
seed=config.engine.execution.seed,
|
107
110
|
rate_limit=config.rate_limit,
|
108
111
|
wait_for_schema=config.wait_for_schema,
|
109
112
|
cassette_config=config.cassette,
|
@@ -40,7 +40,7 @@ def display_section_name(title: str, separator: str = "=", **kwargs: Any) -> Non
|
|
40
40
|
"""Print section name with separators in terminal with the given title nicely centered."""
|
41
41
|
message = f" {title} ".center(get_terminal_width(), separator)
|
42
42
|
kwargs.setdefault("bold", True)
|
43
|
-
click.
|
43
|
+
click.echo(_style(message, **kwargs))
|
44
44
|
|
45
45
|
|
46
46
|
def bold(option: str) -> str:
|
@@ -58,12 +58,14 @@ def display_failures(ctx: ExecutionContext) -> None:
|
|
58
58
|
|
59
59
|
|
60
60
|
if IO_ENCODING != "utf-8":
|
61
|
+
HEADER_SEPARATOR = "-"
|
61
62
|
|
62
63
|
def _style(text: str, **kwargs: Any) -> str:
|
63
64
|
text = text.encode(IO_ENCODING, errors="replace").decode("utf-8")
|
64
65
|
return click.style(text, **kwargs)
|
65
66
|
|
66
67
|
else:
|
68
|
+
HEADER_SEPARATOR = "━"
|
67
69
|
|
68
70
|
def _style(text: str, **kwargs: Any) -> str:
|
69
71
|
return click.style(text, **kwargs)
|
@@ -122,14 +124,14 @@ def _display_extras(extras: list[str]) -> None:
|
|
122
124
|
if extras:
|
123
125
|
click.echo()
|
124
126
|
for extra in extras:
|
125
|
-
click.
|
127
|
+
click.echo(_style(f" {extra}"))
|
126
128
|
|
127
129
|
|
128
130
|
def display_header(version: str) -> None:
|
129
131
|
prefix = "v" if version != "dev" else ""
|
130
132
|
header = f"Schemathesis {prefix}{version}"
|
131
|
-
click.
|
132
|
-
click.
|
133
|
+
click.echo(_style(header, bold=True))
|
134
|
+
click.echo(_style(HEADER_SEPARATOR * len(header), bold=True))
|
133
135
|
click.echo()
|
134
136
|
|
135
137
|
|
@@ -558,6 +560,7 @@ class StatefulProgressManager:
|
|
558
560
|
|
559
561
|
console: Console
|
560
562
|
title: str
|
563
|
+
links_selected: int
|
561
564
|
links_total: int
|
562
565
|
start_time: float
|
563
566
|
|
@@ -574,13 +577,14 @@ class StatefulProgressManager:
|
|
574
577
|
|
575
578
|
# State
|
576
579
|
scenarios: int
|
577
|
-
|
580
|
+
links_covered: set[str]
|
578
581
|
stats: dict[Status, int]
|
579
582
|
is_interrupted: bool
|
580
583
|
|
581
584
|
__slots__ = (
|
582
585
|
"console",
|
583
586
|
"title",
|
587
|
+
"links_selected",
|
584
588
|
"links_total",
|
585
589
|
"start_time",
|
586
590
|
"title_progress",
|
@@ -591,17 +595,18 @@ class StatefulProgressManager:
|
|
591
595
|
"progress_task_id",
|
592
596
|
"stats_task_id",
|
593
597
|
"scenarios",
|
594
|
-
"
|
598
|
+
"links_covered",
|
595
599
|
"stats",
|
596
600
|
"is_interrupted",
|
597
601
|
)
|
598
602
|
|
599
|
-
def __init__(self, console: Console, title: str, links_total: int) -> None:
|
603
|
+
def __init__(self, *, console: Console, title: str, links_selected: int, links_total: int) -> None:
|
600
604
|
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
601
605
|
from rich.style import Style
|
602
606
|
|
603
607
|
self.console = console
|
604
608
|
self.title = title
|
609
|
+
self.links_selected = links_selected
|
605
610
|
self.links_total = links_total
|
606
611
|
self.start_time = time.monotonic()
|
607
612
|
|
@@ -633,7 +638,7 @@ class StatefulProgressManager:
|
|
633
638
|
|
634
639
|
# Initialize state
|
635
640
|
self.scenarios = 0
|
636
|
-
self.
|
641
|
+
self.links_covered = set()
|
637
642
|
self.stats = {
|
638
643
|
Status.SUCCESS: 0,
|
639
644
|
Status.FAILURE: 0,
|
@@ -650,7 +655,9 @@ class StatefulProgressManager:
|
|
650
655
|
|
651
656
|
# Initialize progress displays
|
652
657
|
self.title_task_id = self.title_progress.add_task("Stateful tests")
|
653
|
-
self.progress_task_id = self.progress_bar.add_task(
|
658
|
+
self.progress_task_id = self.progress_bar.add_task(
|
659
|
+
"", scenarios=0, links=f"0 covered / {self.links_selected} selected / {self.links_total} total links"
|
660
|
+
)
|
654
661
|
|
655
662
|
# Create live display
|
656
663
|
group = Group(
|
@@ -668,10 +675,10 @@ class StatefulProgressManager:
|
|
668
675
|
if self.live:
|
669
676
|
self.live.stop()
|
670
677
|
|
671
|
-
def update(self,
|
678
|
+
def update(self, links_covered: set[str], status: Status | None = None) -> None:
|
672
679
|
"""Update progress and stats."""
|
673
680
|
self.scenarios += 1
|
674
|
-
self.
|
681
|
+
self.links_covered.update(links_covered)
|
675
682
|
|
676
683
|
if status is not None:
|
677
684
|
self.stats[status] += 1
|
@@ -685,7 +692,7 @@ class StatefulProgressManager:
|
|
685
692
|
self.progress_bar.update(
|
686
693
|
self.progress_task_id,
|
687
694
|
scenarios=self.scenarios,
|
688
|
-
links=f"{len(self.
|
695
|
+
links=f"{len(self.links_covered)} covered / {self.links_selected} selected / {self.links_total} total links",
|
689
696
|
)
|
690
697
|
|
691
698
|
def _get_stats_message(self) -> str:
|
@@ -770,6 +777,8 @@ def format_duration(duration_ms: int) -> str:
|
|
770
777
|
@dataclass
|
771
778
|
class OutputHandler(EventHandler):
|
772
779
|
workers_num: int
|
780
|
+
# Seed can't be absent in the deterministic mode
|
781
|
+
seed: int | None
|
773
782
|
rate_limit: str | None
|
774
783
|
wait_for_schema: float | None
|
775
784
|
|
@@ -892,6 +901,7 @@ class OutputHandler(EventHandler):
|
|
892
901
|
self.stateful_tests_manager = StatefulProgressManager(
|
893
902
|
console=self.console,
|
894
903
|
title="Stateful tests",
|
904
|
+
links_selected=self.statistic.links.selected,
|
895
905
|
links_total=self.statistic.links.total,
|
896
906
|
)
|
897
907
|
self.stateful_tests_manager.start()
|
@@ -984,7 +994,8 @@ class OutputHandler(EventHandler):
|
|
984
994
|
table.add_column("Value", style="cyan")
|
985
995
|
table.add_row("Scenarios:", f"{self.stateful_tests_manager.scenarios}")
|
986
996
|
table.add_row(
|
987
|
-
"API Links:",
|
997
|
+
"API Links:",
|
998
|
+
f"{len(self.stateful_tests_manager.links_covered)} covered / {self.stateful_tests_manager.links_selected} selected / {self.stateful_tests_manager.links_total} total",
|
988
999
|
)
|
989
1000
|
|
990
1001
|
self.console.print()
|
@@ -1081,7 +1092,7 @@ class OutputHandler(EventHandler):
|
|
1081
1092
|
if not (event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.wait_for_schema is not None):
|
1082
1093
|
suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
|
1083
1094
|
if suggestion is not None:
|
1084
|
-
click.
|
1095
|
+
click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
1085
1096
|
|
1086
1097
|
raise click.Abort
|
1087
1098
|
title = "Test Execution Error"
|
@@ -1089,16 +1100,16 @@ class OutputHandler(EventHandler):
|
|
1089
1100
|
traceback = format_exception(event.exception, with_traceback=True)
|
1090
1101
|
extras = split_traceback(traceback)
|
1091
1102
|
suggestion = f"Please consider reporting the traceback above to our issue tracker:\n\n {ISSUE_TRACKER_URL}."
|
1092
|
-
click.
|
1103
|
+
click.echo(_style(title, fg="red", bold=True))
|
1093
1104
|
click.echo()
|
1094
|
-
click.
|
1105
|
+
click.echo(message)
|
1095
1106
|
_display_extras(extras)
|
1096
1107
|
if not (
|
1097
1108
|
isinstance(event.exception, LoaderError)
|
1098
1109
|
and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
|
1099
1110
|
and self.wait_for_schema is not None
|
1100
1111
|
):
|
1101
|
-
click.
|
1112
|
+
click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
1102
1113
|
|
1103
1114
|
raise click.Abort
|
1104
1115
|
|
@@ -1106,29 +1117,33 @@ class OutputHandler(EventHandler):
|
|
1106
1117
|
display_section_name("WARNINGS")
|
1107
1118
|
total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
|
1108
1119
|
suffix = "" if total == 1 else "s"
|
1109
|
-
click.
|
1110
|
-
|
1111
|
-
|
1120
|
+
click.echo(
|
1121
|
+
_style(
|
1122
|
+
f"\nMissing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
|
1123
|
+
fg="yellow",
|
1124
|
+
)
|
1112
1125
|
)
|
1113
1126
|
|
1114
1127
|
for status_code, operations in self.warnings.missing_auth.items():
|
1115
1128
|
status_text = "Unauthorized" if status_code == 401 else "Forbidden"
|
1116
1129
|
count = len(operations)
|
1117
1130
|
suffix = "" if count == 1 else "s"
|
1118
|
-
click.
|
1119
|
-
|
1120
|
-
|
1131
|
+
click.echo(
|
1132
|
+
_style(
|
1133
|
+
f"{status_code} {status_text} ({count} operation{suffix}):",
|
1134
|
+
fg="yellow",
|
1135
|
+
)
|
1121
1136
|
)
|
1122
1137
|
# Show first few API operations
|
1123
1138
|
for endpoint in operations[:3]:
|
1124
|
-
click.
|
1139
|
+
click.echo(_style(f" - {endpoint}", fg="yellow"))
|
1125
1140
|
if len(operations) > 3:
|
1126
|
-
click.
|
1141
|
+
click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
|
1127
1142
|
click.echo()
|
1128
|
-
click.
|
1129
|
-
click.
|
1130
|
-
click.
|
1131
|
-
click.
|
1143
|
+
click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
|
1144
|
+
click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
|
1145
|
+
click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
|
1146
|
+
click.echo(_style("to provide authentication credentials", fg="yellow"))
|
1132
1147
|
click.echo()
|
1133
1148
|
|
1134
1149
|
def display_experiments(self) -> None:
|
@@ -1136,26 +1151,30 @@ class OutputHandler(EventHandler):
|
|
1136
1151
|
|
1137
1152
|
click.echo()
|
1138
1153
|
for experiment in sorted(GLOBAL_EXPERIMENTS.enabled, key=lambda e: e.name):
|
1139
|
-
click.
|
1140
|
-
click.
|
1141
|
-
click.
|
1154
|
+
click.echo(_style(f"🧪 {experiment.name}: ", bold=True), nl=False)
|
1155
|
+
click.echo(_style(experiment.description))
|
1156
|
+
click.echo(_style(f" Feedback: {experiment.discussion_url}"))
|
1142
1157
|
click.echo()
|
1143
1158
|
|
1144
|
-
click.
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1159
|
+
click.echo(
|
1160
|
+
_style(
|
1161
|
+
"Your feedback is crucial for experimental features. "
|
1162
|
+
"Please visit the provided URL(s) to share your thoughts.",
|
1163
|
+
dim=True,
|
1164
|
+
)
|
1148
1165
|
)
|
1149
1166
|
click.echo()
|
1150
1167
|
|
1151
1168
|
def display_api_operations(self, ctx: ExecutionContext) -> None:
|
1152
1169
|
assert self.statistic is not None
|
1153
|
-
click.
|
1154
|
-
click.
|
1155
|
-
|
1156
|
-
|
1170
|
+
click.echo(_style("API Operations:", bold=True))
|
1171
|
+
click.echo(
|
1172
|
+
_style(
|
1173
|
+
f" Selected: {click.style(str(self.statistic.operations.selected), bold=True)}/"
|
1174
|
+
f"{click.style(str(self.statistic.operations.total), bold=True)}"
|
1175
|
+
)
|
1157
1176
|
)
|
1158
|
-
click.
|
1177
|
+
click.echo(_style(f" Tested: {click.style(str(len(ctx.statistic.tested_operations)), bold=True)}"))
|
1159
1178
|
errors = len(
|
1160
1179
|
{
|
1161
1180
|
err.label
|
@@ -1167,48 +1186,48 @@ class OutputHandler(EventHandler):
|
|
1167
1186
|
}
|
1168
1187
|
)
|
1169
1188
|
if errors:
|
1170
|
-
click.
|
1189
|
+
click.echo(_style(f" Errored: {click.style(str(errors), bold=True)}"))
|
1171
1190
|
|
1172
1191
|
# API operations that are skipped due to fail-fast are counted here as well
|
1173
1192
|
total_skips = self.statistic.operations.selected - len(ctx.statistic.tested_operations) - errors
|
1174
1193
|
if total_skips:
|
1175
|
-
click.
|
1194
|
+
click.echo(_style(f" Skipped: {click.style(str(total_skips), bold=True)}"))
|
1176
1195
|
for reason in sorted(set(self.skip_reasons)):
|
1177
|
-
click.
|
1196
|
+
click.echo(_style(f" - {reason.rstrip('.')}"))
|
1178
1197
|
click.echo()
|
1179
1198
|
|
1180
1199
|
def display_phases(self) -> None:
|
1181
|
-
click.
|
1200
|
+
click.echo(_style("Test Phases:", bold=True))
|
1182
1201
|
|
1183
1202
|
for phase in PhaseName:
|
1184
1203
|
status, skip_reason = self.phases[phase]
|
1185
1204
|
|
1186
1205
|
if status == Status.SKIP:
|
1187
|
-
click.
|
1206
|
+
click.echo(_style(f" ⏭️ {phase.value}", fg="yellow"), nl=False)
|
1188
1207
|
if skip_reason:
|
1189
|
-
click.
|
1208
|
+
click.echo(_style(f" ({skip_reason.value})", fg="yellow"))
|
1190
1209
|
else:
|
1191
1210
|
click.echo()
|
1192
1211
|
elif status == Status.SUCCESS:
|
1193
|
-
click.
|
1212
|
+
click.echo(_style(f" ✅ {phase.value}", fg="green"))
|
1194
1213
|
elif status == Status.FAILURE:
|
1195
|
-
click.
|
1214
|
+
click.echo(_style(f" ❌ {phase.value}", fg="red"))
|
1196
1215
|
elif status == Status.ERROR:
|
1197
|
-
click.
|
1216
|
+
click.echo(_style(f" 🚫 {phase.value}", fg="red"))
|
1198
1217
|
elif status == Status.INTERRUPTED:
|
1199
|
-
click.
|
1218
|
+
click.echo(_style(f" ⚡ {phase.value}", fg="yellow"))
|
1200
1219
|
click.echo()
|
1201
1220
|
|
1202
1221
|
def display_test_cases(self, ctx: ExecutionContext) -> None:
|
1203
1222
|
if ctx.statistic.total_cases == 0:
|
1204
|
-
click.
|
1205
|
-
click.
|
1223
|
+
click.echo(_style("Test cases:", bold=True))
|
1224
|
+
click.echo(" No test cases were generated\n")
|
1206
1225
|
return
|
1207
1226
|
|
1208
1227
|
unique_failures = sum(
|
1209
1228
|
len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
|
1210
1229
|
)
|
1211
|
-
click.
|
1230
|
+
click.echo(_style("Test cases:", bold=True))
|
1212
1231
|
|
1213
1232
|
parts = [f" {click.style(str(ctx.statistic.total_cases), bold=True)} generated"]
|
1214
1233
|
|
@@ -1227,7 +1246,7 @@ class OutputHandler(EventHandler):
|
|
1227
1246
|
if ctx.statistic.cases_without_checks > 0:
|
1228
1247
|
parts.append(f"{click.style(str(ctx.statistic.cases_without_checks), bold=True)} skipped")
|
1229
1248
|
|
1230
|
-
click.
|
1249
|
+
click.echo(_style(", ".join(parts) + "\n"))
|
1231
1250
|
|
1232
1251
|
def display_failures_summary(self, ctx: ExecutionContext) -> None:
|
1233
1252
|
# Collect all unique failures and their counts by title
|
@@ -1238,14 +1257,14 @@ class OutputHandler(EventHandler):
|
|
1238
1257
|
data = failure_counts.get(failure.title, (failure.severity, 0))
|
1239
1258
|
failure_counts[failure.title] = (failure.severity, data[1] + 1)
|
1240
1259
|
|
1241
|
-
click.
|
1260
|
+
click.echo(_style("Failures:", bold=True))
|
1242
1261
|
|
1243
1262
|
# Sort by severity first, then by title
|
1244
1263
|
sorted_failures = sorted(failure_counts.items(), key=lambda x: (x[1][0], x[0]))
|
1245
1264
|
|
1246
1265
|
for title, (_, count) in sorted_failures:
|
1247
|
-
click.
|
1248
|
-
click.
|
1266
|
+
click.echo(_style(f" ❌ {title}: "), nl=False)
|
1267
|
+
click.echo(_style(str(count), bold=True))
|
1249
1268
|
click.echo()
|
1250
1269
|
|
1251
1270
|
def display_errors_summary(self) -> None:
|
@@ -1255,11 +1274,11 @@ class OutputHandler(EventHandler):
|
|
1255
1274
|
title = error.info.title
|
1256
1275
|
error_counts[title] = error_counts.get(title, 0) + 1
|
1257
1276
|
|
1258
|
-
click.
|
1277
|
+
click.echo(_style("Errors:", bold=True))
|
1259
1278
|
|
1260
1279
|
for title in sorted(error_counts):
|
1261
|
-
click.
|
1262
|
-
click.
|
1280
|
+
click.echo(_style(f" 🚫 {title}: "), nl=False)
|
1281
|
+
click.echo(_style(str(error_counts[title]), bold=True))
|
1263
1282
|
click.echo()
|
1264
1283
|
|
1265
1284
|
def display_final_line(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
|
@@ -1269,14 +1288,17 @@ class OutputHandler(EventHandler):
|
|
1269
1288
|
len(group.failures) for grouped in ctx.statistic.failures.values() for group in grouped.values()
|
1270
1289
|
)
|
1271
1290
|
if unique_failures:
|
1272
|
-
|
1291
|
+
suffix = "s" if unique_failures > 1 else ""
|
1292
|
+
parts.append(f"{unique_failures} failure{suffix}")
|
1273
1293
|
|
1274
1294
|
if self.errors:
|
1275
|
-
|
1295
|
+
suffix = "s" if len(self.errors) > 1 else ""
|
1296
|
+
parts.append(f"{len(self.errors)} error{suffix}")
|
1276
1297
|
|
1277
1298
|
total_warnings = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
|
1278
1299
|
if total_warnings:
|
1279
|
-
|
1300
|
+
suffix = "s" if total_warnings > 1 else ""
|
1301
|
+
parts.append(f"{total_warnings} warning{suffix}")
|
1280
1302
|
|
1281
1303
|
if parts:
|
1282
1304
|
message = f"{', '.join(parts)} in {event.running_time:.2f}s"
|
@@ -1299,21 +1321,35 @@ class OutputHandler(EventHandler):
|
|
1299
1321
|
reports.append(("JUnit XML", self.junit_xml_file))
|
1300
1322
|
|
1301
1323
|
if reports:
|
1302
|
-
click.
|
1324
|
+
click.echo(_style("Reports:", bold=True))
|
1303
1325
|
for report_type, path in reports:
|
1304
|
-
click.
|
1326
|
+
click.echo(_style(f" - {report_type}: {path}"))
|
1305
1327
|
click.echo()
|
1306
1328
|
|
1329
|
+
def display_seed(self) -> None:
|
1330
|
+
click.echo(_style("Seed: ", bold=True), nl=False)
|
1331
|
+
if self.seed is None:
|
1332
|
+
click.echo("not used in the deterministic mode")
|
1333
|
+
else:
|
1334
|
+
click.echo(str(self.seed))
|
1335
|
+
click.echo()
|
1336
|
+
|
1307
1337
|
def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
|
1338
|
+
assert self.loading_manager is None
|
1339
|
+
assert self.probing_manager is None
|
1340
|
+
assert self.unit_tests_manager is None
|
1341
|
+
assert self.stateful_tests_manager is None
|
1308
1342
|
if self.errors:
|
1309
1343
|
display_section_name("ERRORS")
|
1310
1344
|
errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label))
|
1311
1345
|
for error in errors:
|
1312
1346
|
display_section_name(error.label, "_", fg="red")
|
1313
1347
|
click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
|
1314
|
-
click.
|
1315
|
-
|
1316
|
-
|
1348
|
+
click.echo(
|
1349
|
+
_style(
|
1350
|
+
f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
|
1351
|
+
fg="red",
|
1352
|
+
)
|
1317
1353
|
)
|
1318
1354
|
display_failures(ctx)
|
1319
1355
|
if self.warnings.missing_auth:
|
@@ -1336,8 +1372,8 @@ class OutputHandler(EventHandler):
|
|
1336
1372
|
|
1337
1373
|
if self.warnings.missing_auth:
|
1338
1374
|
affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
|
1339
|
-
click.
|
1340
|
-
click.
|
1375
|
+
click.echo(_style("Warnings:", bold=True))
|
1376
|
+
click.echo(_style(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
|
1341
1377
|
click.echo()
|
1342
1378
|
|
1343
1379
|
if ctx.summary_lines:
|
@@ -1346,6 +1382,7 @@ class OutputHandler(EventHandler):
|
|
1346
1382
|
|
1347
1383
|
self.display_test_cases(ctx)
|
1348
1384
|
self.display_reports()
|
1385
|
+
self.display_seed()
|
1349
1386
|
self.display_final_line(ctx, event)
|
1350
1387
|
|
1351
1388
|
|
schemathesis/core/errors.py
CHANGED
@@ -102,6 +102,10 @@ class InvalidRegexType(InvalidSchema):
|
|
102
102
|
"""Raised when an invalid type is used where a regex pattern is expected."""
|
103
103
|
|
104
104
|
|
105
|
+
class InvalidLinkDefinition(InvalidSchema):
|
106
|
+
"""Raised when an Open API link references a non-existent operation."""
|
107
|
+
|
108
|
+
|
105
109
|
class MalformedMediaType(ValueError):
|
106
110
|
"""Raised on parsing of incorrect media type."""
|
107
111
|
|
@@ -148,6 +152,10 @@ class IncorrectUsage(SchemathesisError):
|
|
148
152
|
"""Indicates incorrect usage of Schemathesis' public API."""
|
149
153
|
|
150
154
|
|
155
|
+
class NoLinksFound(IncorrectUsage):
|
156
|
+
"""Raised when no valid links are available for stateful testing."""
|
157
|
+
|
158
|
+
|
151
159
|
class InvalidRateLimit(IncorrectUsage):
|
152
160
|
"""Incorrect input for rate limiting."""
|
153
161
|
|
schemathesis/engine/core.py
CHANGED
schemathesis/engine/errors.py
CHANGED
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Callable, Iterator, Sequence, cast
|
|
14
14
|
from schemathesis import errors
|
15
15
|
from schemathesis.core.errors import (
|
16
16
|
RECURSIVE_REFERENCE_ERROR_MESSAGE,
|
17
|
+
InvalidLinkDefinition,
|
17
18
|
SerializationNotPossible,
|
18
19
|
format_exception,
|
19
20
|
get_request_error_extras,
|
@@ -76,6 +77,9 @@ class EngineErrorInfo:
|
|
76
77
|
"""A general error description."""
|
77
78
|
import requests
|
78
79
|
|
80
|
+
if isinstance(self._error, InvalidLinkDefinition):
|
81
|
+
return "Invalid Link Definition"
|
82
|
+
|
79
83
|
if isinstance(self._error, requests.RequestException):
|
80
84
|
return "Network Error"
|
81
85
|
|
@@ -96,6 +100,7 @@ class EngineErrorInfo:
|
|
96
100
|
|
97
101
|
return {
|
98
102
|
RuntimeErrorKind.SCHEMA_UNSUPPORTED: "Unsupported Schema",
|
103
|
+
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Missing Open API links",
|
99
104
|
RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Unknown GraphQL Scalar",
|
100
105
|
RuntimeErrorKind.SERIALIZATION_UNBOUNDED_PREFIX: "XML serialization error",
|
101
106
|
RuntimeErrorKind.SERIALIZATION_NOT_POSSIBLE: "Serialization not possible",
|
@@ -166,6 +171,7 @@ class EngineErrorInfo:
|
|
166
171
|
RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION,
|
167
172
|
RuntimeErrorKind.SCHEMA_UNSUPPORTED,
|
168
173
|
RuntimeErrorKind.SCHEMA_GENERIC,
|
174
|
+
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND,
|
169
175
|
RuntimeErrorKind.SERIALIZATION_NOT_POSSIBLE,
|
170
176
|
RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR,
|
171
177
|
RuntimeErrorKind.HYPOTHESIS_UNSATISFIABLE,
|
@@ -184,11 +190,7 @@ class EngineErrorInfo:
|
|
184
190
|
"""Format error message with optional styling and traceback."""
|
185
191
|
message = []
|
186
192
|
|
187
|
-
|
188
|
-
if self._kind == RuntimeErrorKind.SCHEMA_GENERIC:
|
189
|
-
title = "Schema Error"
|
190
|
-
else:
|
191
|
-
title = self.title
|
193
|
+
title = self.title
|
192
194
|
if title:
|
193
195
|
message.append(f"{title}\n")
|
194
196
|
|
@@ -246,6 +248,7 @@ def get_runtime_error_suggestion(error_type: RuntimeErrorKind, bold: Callable[[s
|
|
246
248
|
return {
|
247
249
|
RuntimeErrorKind.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}.",
|
248
250
|
RuntimeErrorKind.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.",
|
251
|
+
RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Review your endpoint filters to include linked operations",
|
249
252
|
RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax.\n"
|
250
253
|
"For guidance, visit: https://docs.python.org/3/library/re.html",
|
251
254
|
RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Define a custom strategy for it.\n"
|
@@ -299,6 +302,7 @@ class RuntimeErrorKind(str, enum.Enum):
|
|
299
302
|
HYPOTHESIS_HEALTH_CHECK_LARGE_BASE_EXAMPLE = "hypothesis_health_check_large_base_example"
|
300
303
|
|
301
304
|
SCHEMA_INVALID_REGULAR_EXPRESSION = "schema_invalid_regular_expression"
|
305
|
+
SCHEMA_NO_LINKS_FOUND = "schema_no_links_found"
|
302
306
|
SCHEMA_UNSUPPORTED = "schema_unsupported"
|
303
307
|
SCHEMA_GENERIC = "schema_generic"
|
304
308
|
|
@@ -350,6 +354,8 @@ def _classify(*, error: Exception) -> RuntimeErrorKind:
|
|
350
354
|
if isinstance(error, errors.InvalidRegexPattern):
|
351
355
|
return RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION
|
352
356
|
return RuntimeErrorKind.SCHEMA_GENERIC
|
357
|
+
if isinstance(error, errors.NoLinksFound):
|
358
|
+
return RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND
|
353
359
|
if isinstance(error, UnsupportedRecursiveReference):
|
354
360
|
# Recursive references are not supported right now
|
355
361
|
return RuntimeErrorKind.SCHEMA_UNSUPPORTED
|
@@ -20,6 +20,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
|
|
20
20
|
state_machine = engine.schema.as_state_machine()
|
21
21
|
except Exception as exc:
|
22
22
|
yield events.NonFatalError(error=exc, phase=phase.name, label="Stateful tests", related_to_operation=False)
|
23
|
+
yield events.PhaseFinished(phase=phase, status=Status.ERROR, payload=None)
|
23
24
|
return
|
24
25
|
|
25
26
|
event_queue: queue.Queue = queue.Queue()
|