schemathesis 4.0.0a10__py3-none-any.whl → 4.0.0a12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/__init__.py +29 -30
- schemathesis/auths.py +65 -24
- schemathesis/checks.py +73 -39
- schemathesis/cli/commands/__init__.py +51 -3
- schemathesis/cli/commands/data.py +10 -0
- schemathesis/cli/commands/run/__init__.py +163 -274
- schemathesis/cli/commands/run/context.py +8 -4
- schemathesis/cli/commands/run/events.py +11 -1
- schemathesis/cli/commands/run/executor.py +70 -78
- schemathesis/cli/commands/run/filters.py +15 -165
- schemathesis/cli/commands/run/handlers/cassettes.py +105 -104
- schemathesis/cli/commands/run/handlers/junitxml.py +5 -4
- schemathesis/cli/commands/run/handlers/output.py +195 -121
- schemathesis/cli/commands/run/loaders.py +35 -50
- schemathesis/cli/commands/run/validation.py +52 -162
- schemathesis/cli/core.py +5 -3
- schemathesis/cli/ext/fs.py +7 -5
- schemathesis/cli/ext/options.py +0 -21
- schemathesis/config/__init__.py +189 -0
- schemathesis/config/_auth.py +51 -0
- schemathesis/config/_checks.py +268 -0
- schemathesis/config/_diff_base.py +99 -0
- schemathesis/config/_env.py +21 -0
- schemathesis/config/_error.py +156 -0
- schemathesis/config/_generation.py +149 -0
- schemathesis/config/_health_check.py +24 -0
- schemathesis/config/_operations.py +327 -0
- schemathesis/config/_output.py +171 -0
- schemathesis/config/_parameters.py +19 -0
- schemathesis/config/_phases.py +187 -0
- schemathesis/config/_projects.py +523 -0
- schemathesis/config/_rate_limit.py +17 -0
- schemathesis/config/_report.py +120 -0
- schemathesis/config/_validator.py +9 -0
- schemathesis/config/_warnings.py +25 -0
- schemathesis/config/schema.json +885 -0
- schemathesis/core/__init__.py +2 -0
- schemathesis/core/compat.py +16 -9
- schemathesis/core/errors.py +24 -4
- schemathesis/core/failures.py +6 -7
- schemathesis/core/hooks.py +20 -0
- schemathesis/core/output/__init__.py +14 -37
- schemathesis/core/output/sanitization.py +3 -146
- schemathesis/core/transport.py +36 -1
- schemathesis/core/validation.py +16 -0
- schemathesis/engine/__init__.py +2 -4
- schemathesis/engine/context.py +42 -43
- schemathesis/engine/core.py +7 -5
- schemathesis/engine/errors.py +60 -1
- schemathesis/engine/events.py +10 -2
- schemathesis/engine/phases/__init__.py +10 -0
- schemathesis/engine/phases/probes.py +11 -8
- schemathesis/engine/phases/stateful/__init__.py +2 -1
- schemathesis/engine/phases/stateful/_executor.py +104 -46
- schemathesis/engine/phases/stateful/context.py +2 -2
- schemathesis/engine/phases/unit/__init__.py +23 -15
- schemathesis/engine/phases/unit/_executor.py +110 -21
- schemathesis/engine/phases/unit/_pool.py +1 -1
- schemathesis/errors.py +2 -0
- schemathesis/filters.py +2 -3
- schemathesis/generation/__init__.py +5 -33
- schemathesis/generation/case.py +6 -3
- schemathesis/generation/coverage.py +154 -124
- schemathesis/generation/hypothesis/builder.py +70 -20
- schemathesis/generation/meta.py +3 -3
- schemathesis/generation/metrics.py +93 -0
- schemathesis/generation/modes.py +0 -8
- schemathesis/generation/overrides.py +37 -1
- schemathesis/generation/stateful/__init__.py +4 -0
- schemathesis/generation/stateful/state_machine.py +9 -1
- schemathesis/graphql/loaders.py +159 -16
- schemathesis/hooks.py +62 -35
- schemathesis/openapi/checks.py +12 -8
- schemathesis/openapi/generation/filters.py +10 -8
- schemathesis/openapi/loaders.py +142 -17
- schemathesis/pytest/lazy.py +2 -5
- schemathesis/pytest/loaders.py +24 -0
- schemathesis/pytest/plugin.py +33 -2
- schemathesis/schemas.py +21 -66
- schemathesis/specs/graphql/scalars.py +37 -3
- schemathesis/specs/graphql/schemas.py +23 -18
- schemathesis/specs/openapi/_hypothesis.py +26 -28
- schemathesis/specs/openapi/checks.py +37 -36
- schemathesis/specs/openapi/examples.py +4 -3
- schemathesis/specs/openapi/formats.py +32 -5
- schemathesis/specs/openapi/media_types.py +44 -1
- schemathesis/specs/openapi/negative/__init__.py +2 -2
- schemathesis/specs/openapi/patterns.py +46 -16
- schemathesis/specs/openapi/references.py +2 -3
- schemathesis/specs/openapi/schemas.py +19 -22
- schemathesis/specs/openapi/stateful/__init__.py +12 -6
- schemathesis/transport/__init__.py +54 -16
- schemathesis/transport/prepare.py +38 -13
- schemathesis/transport/requests.py +12 -9
- schemathesis/transport/wsgi.py +11 -12
- {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/METADATA +50 -97
- schemathesis-4.0.0a12.dist-info/RECORD +164 -0
- schemathesis/cli/commands/run/checks.py +0 -79
- schemathesis/cli/commands/run/hypothesis.py +0 -78
- schemathesis/cli/commands/run/reports.py +0 -72
- schemathesis/cli/hooks.py +0 -36
- schemathesis/contrib/__init__.py +0 -9
- schemathesis/contrib/openapi/__init__.py +0 -9
- schemathesis/contrib/openapi/fill_missing_examples.py +0 -20
- schemathesis/engine/config.py +0 -59
- schemathesis/experimental/__init__.py +0 -72
- schemathesis/generation/targets.py +0 -69
- schemathesis-4.0.0a10.dist-info/RECORD +0 -153
- {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/licenses/LICENSE +0 -0
@@ -4,6 +4,7 @@ import os
|
|
4
4
|
import textwrap
|
5
5
|
import time
|
6
6
|
from dataclasses import dataclass, field
|
7
|
+
from itertools import groupby
|
7
8
|
from json.decoder import JSONDecodeError
|
8
9
|
from types import GeneratorType
|
9
10
|
from typing import TYPE_CHECKING, Any, Generator, Iterable
|
@@ -13,21 +14,19 @@ import click
|
|
13
14
|
from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
|
14
15
|
from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
|
15
16
|
from schemathesis.cli.commands.run.handlers.base import EventHandler
|
16
|
-
from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
|
17
17
|
from schemathesis.cli.constants import ISSUE_TRACKER_URL
|
18
18
|
from schemathesis.cli.core import get_terminal_width
|
19
|
+
from schemathesis.config import ProjectConfig, ReportFormat, SchemathesisWarning
|
19
20
|
from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
|
20
21
|
from schemathesis.core.failures import MessageBlock, Severity, format_failures
|
21
22
|
from schemathesis.core.output import prepare_response_payload
|
22
23
|
from schemathesis.core.result import Err, Ok
|
23
24
|
from schemathesis.core.version import SCHEMATHESIS_VERSION
|
24
25
|
from schemathesis.engine import Status, events
|
25
|
-
from schemathesis.engine.config import EngineConfig
|
26
26
|
from schemathesis.engine.errors import EngineErrorInfo
|
27
27
|
from schemathesis.engine.phases import PhaseName, PhaseSkipReason
|
28
28
|
from schemathesis.engine.phases.probes import ProbeOutcome
|
29
29
|
from schemathesis.engine.recorder import Interaction, ScenarioRecorder
|
30
|
-
from schemathesis.experimental import GLOBAL_EXPERIMENTS
|
31
30
|
from schemathesis.generation.modes import GenerationMode
|
32
31
|
from schemathesis.schemas import ApiStatistic
|
33
32
|
|
@@ -100,7 +99,7 @@ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks:
|
|
100
99
|
failures=group.failures,
|
101
100
|
curl=group.code_sample,
|
102
101
|
formatter=failure_formatter,
|
103
|
-
config=ctx.
|
102
|
+
config=ctx.config.output,
|
104
103
|
)
|
105
104
|
)
|
106
105
|
click.echo()
|
@@ -326,7 +325,13 @@ class ProbingProgressManager:
|
|
326
325
|
@dataclass
|
327
326
|
class WarningData:
|
328
327
|
missing_auth: dict[int, set[str]] = field(default_factory=dict)
|
329
|
-
|
328
|
+
missing_test_data: set[str] = field(default_factory=set)
|
329
|
+
# operations that only returned 4xx
|
330
|
+
validation_mismatch: set[str] = field(default_factory=set)
|
331
|
+
|
332
|
+
@property
|
333
|
+
def is_empty(self) -> bool:
|
334
|
+
return not bool(self.missing_auth or self.missing_test_data or self.validation_mismatch)
|
330
335
|
|
331
336
|
|
332
337
|
@dataclass
|
@@ -770,12 +775,7 @@ def format_duration(duration_ms: int) -> str:
|
|
770
775
|
|
771
776
|
@dataclass
|
772
777
|
class OutputHandler(EventHandler):
|
773
|
-
|
774
|
-
# Seed can be absent in the deterministic mode
|
775
|
-
seed: int | None
|
776
|
-
rate_limit: str | None
|
777
|
-
wait_for_schema: float | None
|
778
|
-
engine_config: EngineConfig
|
778
|
+
config: ProjectConfig
|
779
779
|
|
780
780
|
loading_manager: LoadingProgressManager | None = None
|
781
781
|
probing_manager: ProbingProgressManager | None = None
|
@@ -784,7 +784,6 @@ class OutputHandler(EventHandler):
|
|
784
784
|
|
785
785
|
statistic: ApiStatistic | None = None
|
786
786
|
skip_reasons: list[str] = field(default_factory=list)
|
787
|
-
report_config: ReportConfig | None = None
|
788
787
|
warnings: WarningData = field(default_factory=WarningData)
|
789
788
|
errors: set[events.NonFatalError] = field(default_factory=set)
|
790
789
|
phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
|
@@ -800,7 +799,7 @@ class OutputHandler(EventHandler):
|
|
800
799
|
elif isinstance(event, events.ScenarioStarted):
|
801
800
|
self._on_scenario_started(event)
|
802
801
|
elif isinstance(event, events.ScenarioFinished):
|
803
|
-
self._on_scenario_finished(event)
|
802
|
+
self._on_scenario_finished(ctx, event)
|
804
803
|
if isinstance(event, events.EngineFinished):
|
805
804
|
self._on_engine_finished(ctx, event)
|
806
805
|
elif isinstance(event, events.Interrupted):
|
@@ -836,6 +835,8 @@ class OutputHandler(EventHandler):
|
|
836
835
|
from rich.style import Style
|
837
836
|
from rich.table import Table
|
838
837
|
|
838
|
+
self.config = event.config
|
839
|
+
|
839
840
|
assert self.loading_manager is not None
|
840
841
|
self.loading_manager.stop()
|
841
842
|
|
@@ -1020,7 +1021,7 @@ class OutputHandler(EventHandler):
|
|
1020
1021
|
assert self.unit_tests_manager is not None
|
1021
1022
|
self.unit_tests_manager.start_operation(event.label)
|
1022
1023
|
|
1023
|
-
def _on_scenario_finished(self, event: events.ScenarioFinished) -> None:
|
1024
|
+
def _on_scenario_finished(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
|
1024
1025
|
if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
|
1025
1026
|
assert self.unit_tests_manager is not None
|
1026
1027
|
if event.label:
|
@@ -1029,7 +1030,7 @@ class OutputHandler(EventHandler):
|
|
1029
1030
|
self.unit_tests_manager.update_stats(event.status)
|
1030
1031
|
if event.status == Status.SKIP and event.skip_reason is not None:
|
1031
1032
|
self.skip_reasons.append(event.skip_reason)
|
1032
|
-
self._check_warnings(event)
|
1033
|
+
self._check_warnings(ctx, event)
|
1033
1034
|
elif (
|
1034
1035
|
event.phase == PhaseName.STATEFUL_TESTING
|
1035
1036
|
and not event.is_final
|
@@ -1039,15 +1040,22 @@ class OutputHandler(EventHandler):
|
|
1039
1040
|
links_seen = {case.transition.id for case in event.recorder.cases.values() if case.transition is not None}
|
1040
1041
|
self.stateful_tests_manager.update(links_seen, event.status)
|
1041
1042
|
|
1042
|
-
def _check_warnings(self, event: events.ScenarioFinished) -> None:
|
1043
|
+
def _check_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
|
1043
1044
|
statistic = aggregate_status_codes(event.recorder.interactions.values())
|
1044
1045
|
|
1045
1046
|
if statistic.total == 0:
|
1046
1047
|
return
|
1047
1048
|
|
1048
|
-
|
1049
|
-
|
1050
|
-
|
1049
|
+
assert ctx.find_operation_by_label is not None
|
1050
|
+
assert event.label is not None
|
1051
|
+
operation = ctx.find_operation_by_label(event.label)
|
1052
|
+
|
1053
|
+
warnings = self.config.warnings_for(operation=operation)
|
1054
|
+
|
1055
|
+
if SchemathesisWarning.MISSING_AUTH in warnings:
|
1056
|
+
for status_code in (401, 403):
|
1057
|
+
if statistic.ratio_for(status_code) >= AUTH_ERRORS_THRESHOLD:
|
1058
|
+
self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
|
1051
1059
|
|
1052
1060
|
# Warn if all positive test cases got 4xx in return and no failure was found
|
1053
1061
|
def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
|
@@ -1068,11 +1076,19 @@ class OutputHandler(EventHandler):
|
|
1068
1076
|
|
1069
1077
|
if (
|
1070
1078
|
event.status == Status.SUCCESS
|
1071
|
-
and
|
1079
|
+
and (
|
1080
|
+
SchemathesisWarning.MISSING_TEST_DATA in warnings or SchemathesisWarning.VALIDATION_MISMATCH in warnings
|
1081
|
+
)
|
1082
|
+
and GenerationMode.POSITIVE in self.config.generation_for(operation=operation, phase=event.phase.name).modes
|
1072
1083
|
and all_positive_are_rejected(event.recorder)
|
1073
|
-
and statistic.should_warn_about_only_4xx()
|
1074
1084
|
):
|
1075
|
-
|
1085
|
+
if SchemathesisWarning.MISSING_TEST_DATA in warnings and statistic.should_warn_about_missing_test_data():
|
1086
|
+
self.warnings.missing_test_data.add(event.recorder.label)
|
1087
|
+
if (
|
1088
|
+
SchemathesisWarning.VALIDATION_MISMATCH in warnings
|
1089
|
+
and statistic.should_warn_about_validation_mismatch()
|
1090
|
+
):
|
1091
|
+
self.warnings.validation_mismatch.add(event.recorder.label)
|
1076
1092
|
|
1077
1093
|
def _on_interrupted(self, event: events.Interrupted) -> None:
|
1078
1094
|
from rich.padding import Padding
|
@@ -1097,6 +1113,7 @@ class OutputHandler(EventHandler):
|
|
1097
1113
|
)
|
1098
1114
|
self.console.print(message)
|
1099
1115
|
self.console.print()
|
1116
|
+
self.probing_manager = None
|
1100
1117
|
|
1101
1118
|
def _on_fatal_error(self, ctx: ExecutionContext, event: events.FatalError) -> None:
|
1102
1119
|
from rich.padding import Padding
|
@@ -1116,7 +1133,9 @@ class OutputHandler(EventHandler):
|
|
1116
1133
|
self.console.print(Padding(Text(extra), (0, 0, 0, 5)))
|
1117
1134
|
self.console.print()
|
1118
1135
|
|
1119
|
-
if not (
|
1136
|
+
if not (
|
1137
|
+
event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.config.wait_for_schema is not None
|
1138
|
+
):
|
1120
1139
|
suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
|
1121
1140
|
if suggestion is not None:
|
1122
1141
|
click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
@@ -1134,85 +1153,83 @@ class OutputHandler(EventHandler):
|
|
1134
1153
|
if not (
|
1135
1154
|
isinstance(event.exception, LoaderError)
|
1136
1155
|
and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
|
1137
|
-
and self.wait_for_schema is not None
|
1156
|
+
and self.config.wait_for_schema is not None
|
1138
1157
|
):
|
1139
1158
|
click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
|
1140
1159
|
|
1141
1160
|
raise click.Abort
|
1142
1161
|
|
1143
|
-
def
|
1144
|
-
|
1145
|
-
|
1146
|
-
if
|
1147
|
-
total = sum(len(
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1162
|
+
def _display_warning_block(
|
1163
|
+
self, title: str, operations: set[str] | dict, tips: list[str], operation_suffix: str = ""
|
1164
|
+
) -> None:
|
1165
|
+
if isinstance(operations, dict):
|
1166
|
+
total = sum(len(ops) for ops in operations.values())
|
1167
|
+
else:
|
1168
|
+
total = len(operations)
|
1169
|
+
|
1170
|
+
suffix = "" if total == 1 else "s"
|
1171
|
+
click.echo(
|
1172
|
+
_style(
|
1173
|
+
f"{title}: {total} operation{suffix}{operation_suffix}\n",
|
1174
|
+
fg="yellow",
|
1154
1175
|
)
|
1176
|
+
)
|
1155
1177
|
|
1156
|
-
|
1178
|
+
# Print up to 3 endpoints, then "+N more"
|
1179
|
+
def _print_up_to_three(operations_: list[str] | set[str]) -> None:
|
1180
|
+
for operation in sorted(operations_)[:3]:
|
1181
|
+
click.echo(_style(f" - {operation}", fg="yellow"))
|
1182
|
+
extra_count = len(operations_) - 3
|
1183
|
+
if extra_count > 0:
|
1184
|
+
click.echo(_style(f" + {extra_count} more", fg="yellow"))
|
1185
|
+
|
1186
|
+
if isinstance(operations, dict):
|
1187
|
+
for status_code, ops in operations.items():
|
1157
1188
|
status_text = "Unauthorized" if status_code == 401 else "Forbidden"
|
1158
|
-
count = len(
|
1189
|
+
count = len(ops)
|
1159
1190
|
suffix = "" if count == 1 else "s"
|
1160
|
-
click.echo(
|
1161
|
-
_style(
|
1162
|
-
f"{status_code} {status_text} ({count} operation{suffix}):",
|
1163
|
-
fg="yellow",
|
1164
|
-
)
|
1165
|
-
)
|
1166
|
-
# Show first few API operations
|
1167
|
-
for endpoint in sorted(operations)[:3]:
|
1168
|
-
click.echo(_style(f" - {endpoint}", fg="yellow"))
|
1169
|
-
if len(operations) > 3:
|
1170
|
-
click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
|
1171
|
-
click.echo()
|
1172
|
-
click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
|
1173
|
-
click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
|
1174
|
-
click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
|
1175
|
-
click.echo(_style("to provide authentication credentials", fg="yellow"))
|
1176
|
-
click.echo()
|
1191
|
+
click.echo(_style(f"{status_code} {status_text} ({count} operation{suffix}):", fg="yellow"))
|
1177
1192
|
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
click.echo(
|
1182
|
-
_style(
|
1183
|
-
f"Schemathesis configuration: {count} operation{suffix} returned only 4xx responses during unit tests\n",
|
1184
|
-
fg="yellow",
|
1185
|
-
)
|
1186
|
-
)
|
1193
|
+
_print_up_to_three(ops)
|
1194
|
+
else:
|
1195
|
+
_print_up_to_three(operations)
|
1187
1196
|
|
1188
|
-
|
1189
|
-
click.echo(_style(f" - {endpoint}", fg="yellow"))
|
1190
|
-
if len(self.warnings.only_4xx_responses) > 3:
|
1191
|
-
click.echo(_style(f" + {len(self.warnings.only_4xx_responses) - 3} more", fg="yellow"))
|
1197
|
+
if tips:
|
1192
1198
|
click.echo()
|
1193
1199
|
|
1194
|
-
|
1195
|
-
click.echo(_style(
|
1196
|
-
click.echo()
|
1200
|
+
for tip in tips:
|
1201
|
+
click.echo(_style(tip, fg="yellow"))
|
1197
1202
|
|
1198
|
-
|
1199
|
-
display_section_name("EXPERIMENTS")
|
1203
|
+
click.echo()
|
1200
1204
|
|
1205
|
+
def display_warnings(self) -> None:
|
1206
|
+
display_section_name("WARNINGS")
|
1201
1207
|
click.echo()
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1208
|
+
if self.warnings.missing_auth:
|
1209
|
+
self._display_warning_block(
|
1210
|
+
title="Missing authentication",
|
1211
|
+
operations=self.warnings.missing_auth,
|
1212
|
+
operation_suffix=" returned authentication errors",
|
1213
|
+
tips=["💡 Use --auth or -H to provide authentication credentials"],
|
1214
|
+
)
|
1207
1215
|
|
1208
|
-
|
1209
|
-
|
1210
|
-
"
|
1211
|
-
|
1212
|
-
|
1216
|
+
if self.warnings.missing_test_data:
|
1217
|
+
self._display_warning_block(
|
1218
|
+
title="Missing test data",
|
1219
|
+
operations=self.warnings.missing_test_data,
|
1220
|
+
operation_suffix=" repeatedly returned 404 Not Found, preventing tests from reaching your API's core logic",
|
1221
|
+
tips=[
|
1222
|
+
"💡 Provide realistic parameter values in your config file so tests can access existing resources",
|
1223
|
+
],
|
1224
|
+
)
|
1225
|
+
|
1226
|
+
if self.warnings.validation_mismatch:
|
1227
|
+
self._display_warning_block(
|
1228
|
+
title="Schema validation mismatch",
|
1229
|
+
operations=self.warnings.validation_mismatch,
|
1230
|
+
operation_suffix=" mostly rejected generated data due to validation errors, indicating schema constraints don't match API validation",
|
1231
|
+
tips=["💡 Check your schema constraints - API validation may be stricter than documented"],
|
1213
1232
|
)
|
1214
|
-
)
|
1215
|
-
click.echo()
|
1216
1233
|
|
1217
1234
|
def display_stateful_failures(self, ctx: ExecutionContext) -> None:
|
1218
1235
|
display_section_name("Stateful tests")
|
@@ -1255,7 +1272,7 @@ class OutputHandler(EventHandler):
|
|
1255
1272
|
click.echo(f"\n{indent}<EMPTY>")
|
1256
1273
|
else:
|
1257
1274
|
try:
|
1258
|
-
payload = prepare_response_payload(response.text, config=ctx.
|
1275
|
+
payload = prepare_response_payload(response.text, config=ctx.config.output)
|
1259
1276
|
click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
|
1260
1277
|
except UnicodeDecodeError:
|
1261
1278
|
click.echo(f"\n{indent}<BINARY>")
|
@@ -1410,24 +1427,27 @@ class OutputHandler(EventHandler):
|
|
1410
1427
|
display_section_name(message, fg=color)
|
1411
1428
|
|
1412
1429
|
def display_reports(self) -> None:
|
1413
|
-
|
1414
|
-
|
1415
|
-
(format.value.upper(), self.report_config.get_path(format).name)
|
1416
|
-
for format in ReportFormat
|
1417
|
-
if format in self.report_config.formats
|
1418
|
-
]
|
1419
|
-
|
1430
|
+
reports = self.config.reports
|
1431
|
+
if reports.vcr.enabled or reports.har.enabled or reports.junit.enabled:
|
1420
1432
|
click.echo(_style("Reports:", bold=True))
|
1421
|
-
for
|
1422
|
-
|
1433
|
+
for format, report in (
|
1434
|
+
(ReportFormat.JUNIT, reports.junit),
|
1435
|
+
(ReportFormat.VCR, reports.vcr),
|
1436
|
+
(ReportFormat.HAR, reports.har),
|
1437
|
+
):
|
1438
|
+
if report.enabled:
|
1439
|
+
path = reports.get_path(format)
|
1440
|
+
click.echo(_style(f" - {format.value.upper()}: {path}"))
|
1423
1441
|
click.echo()
|
1424
1442
|
|
1425
1443
|
def display_seed(self) -> None:
|
1426
1444
|
click.echo(_style("Seed: ", bold=True), nl=False)
|
1427
|
-
if
|
1445
|
+
# Deterministic mode can be applied to a subset of tests, but we only care if it is enabled everywhere
|
1446
|
+
# If not everywhere, then the seed matter and should be displayed
|
1447
|
+
if self.config.seed is None or self.config.generation.deterministic:
|
1428
1448
|
click.echo("not used in the deterministic mode")
|
1429
1449
|
else:
|
1430
|
-
click.echo(str(self.seed))
|
1450
|
+
click.echo(str(self.config.seed))
|
1431
1451
|
click.echo()
|
1432
1452
|
|
1433
1453
|
def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
|
@@ -1438,9 +1458,13 @@ class OutputHandler(EventHandler):
|
|
1438
1458
|
if self.errors:
|
1439
1459
|
display_section_name("ERRORS")
|
1440
1460
|
errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
|
1441
|
-
for
|
1442
|
-
display_section_name(
|
1443
|
-
|
1461
|
+
for label, group_errors in groupby(errors, key=lambda r: r.label):
|
1462
|
+
display_section_name(label, "_", fg="red")
|
1463
|
+
_errors = list(group_errors)
|
1464
|
+
for idx, error in enumerate(_errors, 1):
|
1465
|
+
click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
|
1466
|
+
if idx < len(_errors):
|
1467
|
+
click.echo()
|
1444
1468
|
click.echo(
|
1445
1469
|
_style(
|
1446
1470
|
f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
|
@@ -1448,10 +1472,8 @@ class OutputHandler(EventHandler):
|
|
1448
1472
|
)
|
1449
1473
|
)
|
1450
1474
|
display_failures(ctx)
|
1451
|
-
if
|
1475
|
+
if not self.warnings.is_empty:
|
1452
1476
|
self.display_warnings()
|
1453
|
-
if GLOBAL_EXPERIMENTS.enabled:
|
1454
|
-
self.display_experiments()
|
1455
1477
|
if ctx.statistic.extraction_failures:
|
1456
1478
|
self.display_stateful_failures(ctx)
|
1457
1479
|
display_section_name("SUMMARY")
|
@@ -1468,21 +1490,39 @@ class OutputHandler(EventHandler):
|
|
1468
1490
|
if self.errors:
|
1469
1491
|
self.display_errors_summary()
|
1470
1492
|
|
1471
|
-
if
|
1493
|
+
if not self.warnings.is_empty:
|
1472
1494
|
click.echo(_style("Warnings:", bold=True))
|
1473
1495
|
|
1474
1496
|
if self.warnings.missing_auth:
|
1475
1497
|
affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
|
1476
|
-
|
1498
|
+
suffix = "" if affected == 1 else "s"
|
1499
|
+
click.echo(
|
1500
|
+
_style(
|
1501
|
+
f" ⚠️ Missing authentication: {bold(str(affected))} operation{suffix} returned only 401/403 responses",
|
1502
|
+
fg="yellow",
|
1503
|
+
)
|
1504
|
+
)
|
1477
1505
|
|
1478
|
-
if self.warnings.
|
1479
|
-
count = len(self.warnings.
|
1506
|
+
if self.warnings.missing_test_data:
|
1507
|
+
count = len(self.warnings.missing_test_data)
|
1480
1508
|
suffix = "" if count == 1 else "s"
|
1481
1509
|
click.echo(
|
1482
|
-
_style(
|
1483
|
-
|
1510
|
+
_style(
|
1511
|
+
f" ⚠️ Missing valid test data: {bold(str(count))} operation{suffix} repeatedly returned 404 responses",
|
1512
|
+
fg="yellow",
|
1513
|
+
)
|
1514
|
+
)
|
1515
|
+
|
1516
|
+
if self.warnings.validation_mismatch:
|
1517
|
+
count = len(self.warnings.validation_mismatch)
|
1518
|
+
suffix = "" if count == 1 else "s"
|
1519
|
+
click.echo(
|
1520
|
+
_style(
|
1521
|
+
f" ⚠️ Schema validation mismatch: {bold(str(count))} operation{suffix} mostly rejected generated data",
|
1522
|
+
fg="yellow",
|
1523
|
+
)
|
1484
1524
|
)
|
1485
|
-
|
1525
|
+
|
1486
1526
|
click.echo()
|
1487
1527
|
|
1488
1528
|
if ctx.summary_lines:
|
@@ -1495,12 +1535,6 @@ class OutputHandler(EventHandler):
|
|
1495
1535
|
self.display_final_line(ctx, event)
|
1496
1536
|
|
1497
1537
|
|
1498
|
-
TOO_MANY_RESPONSES_WARNING_TEMPLATE = (
|
1499
|
-
"Most of the responses from {} have a {} status code. Did you specify proper API credentials?"
|
1500
|
-
)
|
1501
|
-
TOO_MANY_RESPONSES_THRESHOLD = 0.9
|
1502
|
-
|
1503
|
-
|
1504
1538
|
@dataclass
|
1505
1539
|
class StatusCodeStatistic:
|
1506
1540
|
"""Statistics about HTTP status codes in a scenario."""
|
@@ -1516,15 +1550,55 @@ class StatusCodeStatistic:
|
|
1516
1550
|
return 0.0
|
1517
1551
|
return self.counts.get(status_code, 0) / self.total
|
1518
1552
|
|
1519
|
-
def
|
1520
|
-
"""
|
1553
|
+
def _get_4xx_breakdown(self) -> tuple[int, int, int]:
|
1554
|
+
"""Get breakdown of 4xx responses: (404_count, other_4xx_count, total_4xx_count)."""
|
1555
|
+
count_404 = self.counts.get(404, 0)
|
1556
|
+
count_other_4xx = sum(
|
1557
|
+
count for code, count in self.counts.items() if 400 <= code < 500 and code not in {401, 403, 404}
|
1558
|
+
)
|
1559
|
+
total_4xx = count_404 + count_other_4xx
|
1560
|
+
return count_404, count_other_4xx, total_4xx
|
1561
|
+
|
1562
|
+
def _is_only_4xx_responses(self) -> bool:
|
1563
|
+
"""Check if all responses are 4xx (excluding 5xx)."""
|
1564
|
+
return all(400 <= code < 500 for code in self.counts.keys() if code not in {500})
|
1565
|
+
|
1566
|
+
def _can_warn_about_4xx(self) -> bool:
|
1567
|
+
"""Check basic conditions for 4xx warnings."""
|
1521
1568
|
if self.total == 0:
|
1522
1569
|
return False
|
1523
|
-
#
|
1524
|
-
if set(self.counts.keys()) <= {401, 403}:
|
1570
|
+
# Skip if only auth errors
|
1571
|
+
if set(self.counts.keys()) <= {401, 403, 500}:
|
1572
|
+
return False
|
1573
|
+
return self._is_only_4xx_responses()
|
1574
|
+
|
1575
|
+
def should_warn_about_missing_test_data(self) -> bool:
|
1576
|
+
"""Check if an operation should be warned about missing test data (significant 404 responses)."""
|
1577
|
+
if not self._can_warn_about_4xx():
|
1578
|
+
return False
|
1579
|
+
|
1580
|
+
count_404, _, total_4xx = self._get_4xx_breakdown()
|
1581
|
+
|
1582
|
+
if total_4xx == 0:
|
1583
|
+
return False
|
1584
|
+
|
1585
|
+
return (count_404 / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
|
1586
|
+
|
1587
|
+
def should_warn_about_validation_mismatch(self) -> bool:
|
1588
|
+
"""Check if an operation should be warned about validation mismatch (significant 400/422 responses)."""
|
1589
|
+
if not self._can_warn_about_4xx():
|
1590
|
+
return False
|
1591
|
+
|
1592
|
+
_, count_other_4xx, total_4xx = self._get_4xx_breakdown()
|
1593
|
+
|
1594
|
+
if total_4xx == 0:
|
1525
1595
|
return False
|
1526
|
-
|
1527
|
-
return
|
1596
|
+
|
1597
|
+
return (count_other_4xx / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
|
1598
|
+
|
1599
|
+
|
1600
|
+
AUTH_ERRORS_THRESHOLD = 0.9
|
1601
|
+
OTHER_CLIENT_ERRORS_THRESHOLD = 0.1
|
1528
1602
|
|
1529
1603
|
|
1530
1604
|
def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
|
@@ -6,42 +6,28 @@ supporting both GraphQL and OpenAPI specifications.
|
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import os
|
9
10
|
import warnings
|
10
|
-
from dataclasses import dataclass
|
11
11
|
from typing import TYPE_CHECKING, Any, Callable
|
12
12
|
|
13
13
|
from schemathesis import graphql, openapi
|
14
|
-
from schemathesis.
|
14
|
+
from schemathesis.config import ProjectConfig
|
15
15
|
from schemathesis.core.errors import LoaderError, LoaderErrorKind
|
16
16
|
from schemathesis.core.fs import file_exists
|
17
|
-
from schemathesis.core.output import OutputConfig
|
18
|
-
from schemathesis.generation import GenerationConfig
|
19
17
|
|
20
18
|
if TYPE_CHECKING:
|
21
|
-
from schemathesis.engine.config import NetworkConfig
|
22
19
|
from schemathesis.schemas import BaseSchema
|
23
20
|
|
24
|
-
Loader = Callable[["
|
21
|
+
Loader = Callable[["ProjectConfig"], "BaseSchema"]
|
25
22
|
|
26
23
|
|
27
|
-
|
28
|
-
class AutodetectConfig:
|
29
|
-
location: str
|
30
|
-
network: NetworkConfig
|
31
|
-
wait_for_schema: float | None
|
32
|
-
base_url: str | None | NotSet = NOT_SET
|
33
|
-
rate_limit: str | None | NotSet = NOT_SET
|
34
|
-
generation: GenerationConfig | NotSet = NOT_SET
|
35
|
-
output: OutputConfig | NotSet = NOT_SET
|
36
|
-
|
37
|
-
|
38
|
-
def load_schema(config: AutodetectConfig) -> BaseSchema:
|
24
|
+
def load_schema(location: str, config: ProjectConfig) -> BaseSchema:
|
39
25
|
"""Load API schema automatically based on the provided configuration."""
|
40
|
-
if is_probably_graphql(
|
26
|
+
if is_probably_graphql(location):
|
41
27
|
# Try GraphQL first, then fallback to Open API
|
42
|
-
return _try_load_schema(config, graphql, openapi)
|
28
|
+
return _try_load_schema(location, config, graphql, openapi)
|
43
29
|
# Try Open API first, then fallback to GraphQL
|
44
|
-
return _try_load_schema(config, openapi, graphql)
|
30
|
+
return _try_load_schema(location, config, openapi, graphql)
|
45
31
|
|
46
32
|
|
47
33
|
def should_try_more(exc: LoaderError) -> bool:
|
@@ -60,27 +46,28 @@ def should_try_more(exc: LoaderError) -> bool:
|
|
60
46
|
)
|
61
47
|
|
62
48
|
|
63
|
-
def detect_loader(
|
49
|
+
def detect_loader(location: str, module: Any) -> Callable:
|
64
50
|
"""Detect API schema loader."""
|
65
|
-
if
|
66
|
-
|
67
|
-
|
68
|
-
return module.from_url # type: ignore
|
69
|
-
raise NotImplementedError
|
51
|
+
if file_exists(location):
|
52
|
+
return module.from_path # type: ignore
|
53
|
+
return module.from_url # type: ignore
|
70
54
|
|
71
55
|
|
72
|
-
def _try_load_schema(config:
|
56
|
+
def _try_load_schema(location: str, config: ProjectConfig, first_module: Any, second_module: Any) -> BaseSchema:
|
73
57
|
"""Try to load schema with fallback option."""
|
74
58
|
from urllib3.exceptions import InsecureRequestWarning
|
75
59
|
|
76
60
|
with warnings.catch_warnings():
|
77
61
|
warnings.simplefilter("ignore", InsecureRequestWarning)
|
78
62
|
try:
|
79
|
-
return _load_schema(config, first_module)
|
63
|
+
return _load_schema(location, config, first_module)
|
80
64
|
except LoaderError as exc:
|
65
|
+
# If this was the OpenAPI loader on an explicit OpenAPI file, don't fallback
|
66
|
+
if first_module is openapi and is_openapi_file(location):
|
67
|
+
raise exc
|
81
68
|
if should_try_more(exc):
|
82
69
|
try:
|
83
|
-
return _load_schema(config, second_module)
|
70
|
+
return _load_schema(location, config, second_module)
|
84
71
|
except Exception as second_exc:
|
85
72
|
if is_specific_exception(second_exc):
|
86
73
|
raise second_exc
|
@@ -88,26 +75,23 @@ def _try_load_schema(config: AutodetectConfig, first_module: Any, second_module:
|
|
88
75
|
raise exc
|
89
76
|
|
90
77
|
|
91
|
-
def _load_schema(config:
|
78
|
+
def _load_schema(location: str, config: ProjectConfig, module: Any) -> BaseSchema:
|
92
79
|
"""Unified schema loader for both GraphQL and OpenAPI."""
|
93
|
-
loader = detect_loader(
|
80
|
+
loader = detect_loader(location, module)
|
94
81
|
|
95
82
|
kwargs: dict = {}
|
96
83
|
if loader is module.from_url:
|
97
84
|
if config.wait_for_schema is not None:
|
98
85
|
kwargs["wait_for_schema"] = config.wait_for_schema
|
99
|
-
kwargs["verify"] = config.
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
output=config.output,
|
109
|
-
generation=config.generation,
|
110
|
-
)
|
86
|
+
kwargs["verify"] = config.tls_verify
|
87
|
+
request_cert = config.request_cert_for()
|
88
|
+
if request_cert:
|
89
|
+
kwargs["cert"] = request_cert
|
90
|
+
auth = config.auth_for()
|
91
|
+
if auth is not None:
|
92
|
+
kwargs["auth"] = auth
|
93
|
+
|
94
|
+
return loader(location, config=config._parent, **kwargs)
|
111
95
|
|
112
96
|
|
113
97
|
def is_specific_exception(exc: Exception) -> bool:
|
@@ -120,10 +104,11 @@ def is_specific_exception(exc: Exception) -> bool:
|
|
120
104
|
)
|
121
105
|
|
122
106
|
|
123
|
-
def is_probably_graphql(
|
107
|
+
def is_probably_graphql(location: str) -> bool:
|
124
108
|
"""Detect whether it is likely that the given location is a GraphQL endpoint."""
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
)
|
109
|
+
return location.endswith(("/graphql", "/graphql/", ".graphql", ".gql"))
|
110
|
+
|
111
|
+
|
112
|
+
def is_openapi_file(location: str) -> bool:
|
113
|
+
name = os.path.basename(location).lower()
|
114
|
+
return any(name == f"{base}{ext}" for base in ("openapi", "swagger") for ext in (".json", ".yaml", ".yml"))
|