schemathesis 4.0.0a10__py3-none-any.whl → 4.0.0a12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. schemathesis/__init__.py +29 -30
  2. schemathesis/auths.py +65 -24
  3. schemathesis/checks.py +73 -39
  4. schemathesis/cli/commands/__init__.py +51 -3
  5. schemathesis/cli/commands/data.py +10 -0
  6. schemathesis/cli/commands/run/__init__.py +163 -274
  7. schemathesis/cli/commands/run/context.py +8 -4
  8. schemathesis/cli/commands/run/events.py +11 -1
  9. schemathesis/cli/commands/run/executor.py +70 -78
  10. schemathesis/cli/commands/run/filters.py +15 -165
  11. schemathesis/cli/commands/run/handlers/cassettes.py +105 -104
  12. schemathesis/cli/commands/run/handlers/junitxml.py +5 -4
  13. schemathesis/cli/commands/run/handlers/output.py +195 -121
  14. schemathesis/cli/commands/run/loaders.py +35 -50
  15. schemathesis/cli/commands/run/validation.py +52 -162
  16. schemathesis/cli/core.py +5 -3
  17. schemathesis/cli/ext/fs.py +7 -5
  18. schemathesis/cli/ext/options.py +0 -21
  19. schemathesis/config/__init__.py +189 -0
  20. schemathesis/config/_auth.py +51 -0
  21. schemathesis/config/_checks.py +268 -0
  22. schemathesis/config/_diff_base.py +99 -0
  23. schemathesis/config/_env.py +21 -0
  24. schemathesis/config/_error.py +156 -0
  25. schemathesis/config/_generation.py +149 -0
  26. schemathesis/config/_health_check.py +24 -0
  27. schemathesis/config/_operations.py +327 -0
  28. schemathesis/config/_output.py +171 -0
  29. schemathesis/config/_parameters.py +19 -0
  30. schemathesis/config/_phases.py +187 -0
  31. schemathesis/config/_projects.py +523 -0
  32. schemathesis/config/_rate_limit.py +17 -0
  33. schemathesis/config/_report.py +120 -0
  34. schemathesis/config/_validator.py +9 -0
  35. schemathesis/config/_warnings.py +25 -0
  36. schemathesis/config/schema.json +885 -0
  37. schemathesis/core/__init__.py +2 -0
  38. schemathesis/core/compat.py +16 -9
  39. schemathesis/core/errors.py +24 -4
  40. schemathesis/core/failures.py +6 -7
  41. schemathesis/core/hooks.py +20 -0
  42. schemathesis/core/output/__init__.py +14 -37
  43. schemathesis/core/output/sanitization.py +3 -146
  44. schemathesis/core/transport.py +36 -1
  45. schemathesis/core/validation.py +16 -0
  46. schemathesis/engine/__init__.py +2 -4
  47. schemathesis/engine/context.py +42 -43
  48. schemathesis/engine/core.py +7 -5
  49. schemathesis/engine/errors.py +60 -1
  50. schemathesis/engine/events.py +10 -2
  51. schemathesis/engine/phases/__init__.py +10 -0
  52. schemathesis/engine/phases/probes.py +11 -8
  53. schemathesis/engine/phases/stateful/__init__.py +2 -1
  54. schemathesis/engine/phases/stateful/_executor.py +104 -46
  55. schemathesis/engine/phases/stateful/context.py +2 -2
  56. schemathesis/engine/phases/unit/__init__.py +23 -15
  57. schemathesis/engine/phases/unit/_executor.py +110 -21
  58. schemathesis/engine/phases/unit/_pool.py +1 -1
  59. schemathesis/errors.py +2 -0
  60. schemathesis/filters.py +2 -3
  61. schemathesis/generation/__init__.py +5 -33
  62. schemathesis/generation/case.py +6 -3
  63. schemathesis/generation/coverage.py +154 -124
  64. schemathesis/generation/hypothesis/builder.py +70 -20
  65. schemathesis/generation/meta.py +3 -3
  66. schemathesis/generation/metrics.py +93 -0
  67. schemathesis/generation/modes.py +0 -8
  68. schemathesis/generation/overrides.py +37 -1
  69. schemathesis/generation/stateful/__init__.py +4 -0
  70. schemathesis/generation/stateful/state_machine.py +9 -1
  71. schemathesis/graphql/loaders.py +159 -16
  72. schemathesis/hooks.py +62 -35
  73. schemathesis/openapi/checks.py +12 -8
  74. schemathesis/openapi/generation/filters.py +10 -8
  75. schemathesis/openapi/loaders.py +142 -17
  76. schemathesis/pytest/lazy.py +2 -5
  77. schemathesis/pytest/loaders.py +24 -0
  78. schemathesis/pytest/plugin.py +33 -2
  79. schemathesis/schemas.py +21 -66
  80. schemathesis/specs/graphql/scalars.py +37 -3
  81. schemathesis/specs/graphql/schemas.py +23 -18
  82. schemathesis/specs/openapi/_hypothesis.py +26 -28
  83. schemathesis/specs/openapi/checks.py +37 -36
  84. schemathesis/specs/openapi/examples.py +4 -3
  85. schemathesis/specs/openapi/formats.py +32 -5
  86. schemathesis/specs/openapi/media_types.py +44 -1
  87. schemathesis/specs/openapi/negative/__init__.py +2 -2
  88. schemathesis/specs/openapi/patterns.py +46 -16
  89. schemathesis/specs/openapi/references.py +2 -3
  90. schemathesis/specs/openapi/schemas.py +19 -22
  91. schemathesis/specs/openapi/stateful/__init__.py +12 -6
  92. schemathesis/transport/__init__.py +54 -16
  93. schemathesis/transport/prepare.py +38 -13
  94. schemathesis/transport/requests.py +12 -9
  95. schemathesis/transport/wsgi.py +11 -12
  96. {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/METADATA +50 -97
  97. schemathesis-4.0.0a12.dist-info/RECORD +164 -0
  98. schemathesis/cli/commands/run/checks.py +0 -79
  99. schemathesis/cli/commands/run/hypothesis.py +0 -78
  100. schemathesis/cli/commands/run/reports.py +0 -72
  101. schemathesis/cli/hooks.py +0 -36
  102. schemathesis/contrib/__init__.py +0 -9
  103. schemathesis/contrib/openapi/__init__.py +0 -9
  104. schemathesis/contrib/openapi/fill_missing_examples.py +0 -20
  105. schemathesis/engine/config.py +0 -59
  106. schemathesis/experimental/__init__.py +0 -72
  107. schemathesis/generation/targets.py +0 -69
  108. schemathesis-4.0.0a10.dist-info/RECORD +0 -153
  109. {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/WHEEL +0 -0
  110. {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/entry_points.txt +0 -0
  111. {schemathesis-4.0.0a10.dist-info → schemathesis-4.0.0a12.dist-info}/licenses/LICENSE +0 -0
@@ -4,6 +4,7 @@ import os
4
4
  import textwrap
5
5
  import time
6
6
  from dataclasses import dataclass, field
7
+ from itertools import groupby
7
8
  from json.decoder import JSONDecodeError
8
9
  from types import GeneratorType
9
10
  from typing import TYPE_CHECKING, Any, Generator, Iterable
@@ -13,21 +14,19 @@ import click
13
14
  from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
14
15
  from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
15
16
  from schemathesis.cli.commands.run.handlers.base import EventHandler
16
- from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
17
17
  from schemathesis.cli.constants import ISSUE_TRACKER_URL
18
18
  from schemathesis.cli.core import get_terminal_width
19
+ from schemathesis.config import ProjectConfig, ReportFormat, SchemathesisWarning
19
20
  from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
20
21
  from schemathesis.core.failures import MessageBlock, Severity, format_failures
21
22
  from schemathesis.core.output import prepare_response_payload
22
23
  from schemathesis.core.result import Err, Ok
23
24
  from schemathesis.core.version import SCHEMATHESIS_VERSION
24
25
  from schemathesis.engine import Status, events
25
- from schemathesis.engine.config import EngineConfig
26
26
  from schemathesis.engine.errors import EngineErrorInfo
27
27
  from schemathesis.engine.phases import PhaseName, PhaseSkipReason
28
28
  from schemathesis.engine.phases.probes import ProbeOutcome
29
29
  from schemathesis.engine.recorder import Interaction, ScenarioRecorder
30
- from schemathesis.experimental import GLOBAL_EXPERIMENTS
31
30
  from schemathesis.generation.modes import GenerationMode
32
31
  from schemathesis.schemas import ApiStatistic
33
32
 
@@ -100,7 +99,7 @@ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks:
100
99
  failures=group.failures,
101
100
  curl=group.code_sample,
102
101
  formatter=failure_formatter,
103
- config=ctx.output_config,
102
+ config=ctx.config.output,
104
103
  )
105
104
  )
106
105
  click.echo()
@@ -326,7 +325,13 @@ class ProbingProgressManager:
326
325
  @dataclass
327
326
  class WarningData:
328
327
  missing_auth: dict[int, set[str]] = field(default_factory=dict)
329
- only_4xx_responses: set[str] = field(default_factory=set) # operations that only returned 4xx
328
+ missing_test_data: set[str] = field(default_factory=set)
329
+ # operations that only returned 4xx
330
+ validation_mismatch: set[str] = field(default_factory=set)
331
+
332
+ @property
333
+ def is_empty(self) -> bool:
334
+ return not bool(self.missing_auth or self.missing_test_data or self.validation_mismatch)
330
335
 
331
336
 
332
337
  @dataclass
@@ -770,12 +775,7 @@ def format_duration(duration_ms: int) -> str:
770
775
 
771
776
  @dataclass
772
777
  class OutputHandler(EventHandler):
773
- workers_num: int
774
- # Seed can be absent in the deterministic mode
775
- seed: int | None
776
- rate_limit: str | None
777
- wait_for_schema: float | None
778
- engine_config: EngineConfig
778
+ config: ProjectConfig
779
779
 
780
780
  loading_manager: LoadingProgressManager | None = None
781
781
  probing_manager: ProbingProgressManager | None = None
@@ -784,7 +784,6 @@ class OutputHandler(EventHandler):
784
784
 
785
785
  statistic: ApiStatistic | None = None
786
786
  skip_reasons: list[str] = field(default_factory=list)
787
- report_config: ReportConfig | None = None
788
787
  warnings: WarningData = field(default_factory=WarningData)
789
788
  errors: set[events.NonFatalError] = field(default_factory=set)
790
789
  phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
@@ -800,7 +799,7 @@ class OutputHandler(EventHandler):
800
799
  elif isinstance(event, events.ScenarioStarted):
801
800
  self._on_scenario_started(event)
802
801
  elif isinstance(event, events.ScenarioFinished):
803
- self._on_scenario_finished(event)
802
+ self._on_scenario_finished(ctx, event)
804
803
  if isinstance(event, events.EngineFinished):
805
804
  self._on_engine_finished(ctx, event)
806
805
  elif isinstance(event, events.Interrupted):
@@ -836,6 +835,8 @@ class OutputHandler(EventHandler):
836
835
  from rich.style import Style
837
836
  from rich.table import Table
838
837
 
838
+ self.config = event.config
839
+
839
840
  assert self.loading_manager is not None
840
841
  self.loading_manager.stop()
841
842
 
@@ -1020,7 +1021,7 @@ class OutputHandler(EventHandler):
1020
1021
  assert self.unit_tests_manager is not None
1021
1022
  self.unit_tests_manager.start_operation(event.label)
1022
1023
 
1023
- def _on_scenario_finished(self, event: events.ScenarioFinished) -> None:
1024
+ def _on_scenario_finished(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1024
1025
  if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
1025
1026
  assert self.unit_tests_manager is not None
1026
1027
  if event.label:
@@ -1029,7 +1030,7 @@ class OutputHandler(EventHandler):
1029
1030
  self.unit_tests_manager.update_stats(event.status)
1030
1031
  if event.status == Status.SKIP and event.skip_reason is not None:
1031
1032
  self.skip_reasons.append(event.skip_reason)
1032
- self._check_warnings(event)
1033
+ self._check_warnings(ctx, event)
1033
1034
  elif (
1034
1035
  event.phase == PhaseName.STATEFUL_TESTING
1035
1036
  and not event.is_final
@@ -1039,15 +1040,22 @@ class OutputHandler(EventHandler):
1039
1040
  links_seen = {case.transition.id for case in event.recorder.cases.values() if case.transition is not None}
1040
1041
  self.stateful_tests_manager.update(links_seen, event.status)
1041
1042
 
1042
- def _check_warnings(self, event: events.ScenarioFinished) -> None:
1043
+ def _check_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1043
1044
  statistic = aggregate_status_codes(event.recorder.interactions.values())
1044
1045
 
1045
1046
  if statistic.total == 0:
1046
1047
  return
1047
1048
 
1048
- for status_code in (401, 403):
1049
- if statistic.ratio_for(status_code) >= TOO_MANY_RESPONSES_THRESHOLD:
1050
- self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
1049
+ assert ctx.find_operation_by_label is not None
1050
+ assert event.label is not None
1051
+ operation = ctx.find_operation_by_label(event.label)
1052
+
1053
+ warnings = self.config.warnings_for(operation=operation)
1054
+
1055
+ if SchemathesisWarning.MISSING_AUTH in warnings:
1056
+ for status_code in (401, 403):
1057
+ if statistic.ratio_for(status_code) >= AUTH_ERRORS_THRESHOLD:
1058
+ self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
1051
1059
 
1052
1060
  # Warn if all positive test cases got 4xx in return and no failure was found
1053
1061
  def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
@@ -1068,11 +1076,19 @@ class OutputHandler(EventHandler):
1068
1076
 
1069
1077
  if (
1070
1078
  event.status == Status.SUCCESS
1071
- and GenerationMode.POSITIVE in self.engine_config.execution.generation.modes
1079
+ and (
1080
+ SchemathesisWarning.MISSING_TEST_DATA in warnings or SchemathesisWarning.VALIDATION_MISMATCH in warnings
1081
+ )
1082
+ and GenerationMode.POSITIVE in self.config.generation_for(operation=operation, phase=event.phase.name).modes
1072
1083
  and all_positive_are_rejected(event.recorder)
1073
- and statistic.should_warn_about_only_4xx()
1074
1084
  ):
1075
- self.warnings.only_4xx_responses.add(event.recorder.label)
1085
+ if SchemathesisWarning.MISSING_TEST_DATA in warnings and statistic.should_warn_about_missing_test_data():
1086
+ self.warnings.missing_test_data.add(event.recorder.label)
1087
+ if (
1088
+ SchemathesisWarning.VALIDATION_MISMATCH in warnings
1089
+ and statistic.should_warn_about_validation_mismatch()
1090
+ ):
1091
+ self.warnings.validation_mismatch.add(event.recorder.label)
1076
1092
 
1077
1093
  def _on_interrupted(self, event: events.Interrupted) -> None:
1078
1094
  from rich.padding import Padding
@@ -1097,6 +1113,7 @@ class OutputHandler(EventHandler):
1097
1113
  )
1098
1114
  self.console.print(message)
1099
1115
  self.console.print()
1116
+ self.probing_manager = None
1100
1117
 
1101
1118
  def _on_fatal_error(self, ctx: ExecutionContext, event: events.FatalError) -> None:
1102
1119
  from rich.padding import Padding
@@ -1116,7 +1133,9 @@ class OutputHandler(EventHandler):
1116
1133
  self.console.print(Padding(Text(extra), (0, 0, 0, 5)))
1117
1134
  self.console.print()
1118
1135
 
1119
- if not (event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.wait_for_schema is not None):
1136
+ if not (
1137
+ event.exception.kind == LoaderErrorKind.CONNECTION_OTHER and self.config.wait_for_schema is not None
1138
+ ):
1120
1139
  suggestion = LOADER_ERROR_SUGGESTIONS.get(event.exception.kind)
1121
1140
  if suggestion is not None:
1122
1141
  click.echo(_style(f"{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
@@ -1134,85 +1153,83 @@ class OutputHandler(EventHandler):
1134
1153
  if not (
1135
1154
  isinstance(event.exception, LoaderError)
1136
1155
  and event.exception.kind == LoaderErrorKind.CONNECTION_OTHER
1137
- and self.wait_for_schema is not None
1156
+ and self.config.wait_for_schema is not None
1138
1157
  ):
1139
1158
  click.echo(_style(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}"))
1140
1159
 
1141
1160
  raise click.Abort
1142
1161
 
1143
- def display_warnings(self) -> None:
1144
- display_section_name("WARNINGS")
1145
- click.echo()
1146
- if self.warnings.missing_auth:
1147
- total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
1148
- suffix = "" if total == 1 else "s"
1149
- click.echo(
1150
- _style(
1151
- f"Missing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
1152
- fg="yellow",
1153
- )
1162
+ def _display_warning_block(
1163
+ self, title: str, operations: set[str] | dict, tips: list[str], operation_suffix: str = ""
1164
+ ) -> None:
1165
+ if isinstance(operations, dict):
1166
+ total = sum(len(ops) for ops in operations.values())
1167
+ else:
1168
+ total = len(operations)
1169
+
1170
+ suffix = "" if total == 1 else "s"
1171
+ click.echo(
1172
+ _style(
1173
+ f"{title}: {total} operation{suffix}{operation_suffix}\n",
1174
+ fg="yellow",
1154
1175
  )
1176
+ )
1155
1177
 
1156
- for status_code, operations in self.warnings.missing_auth.items():
1178
+ # Print up to 3 endpoints, then "+N more"
1179
+ def _print_up_to_three(operations_: list[str] | set[str]) -> None:
1180
+ for operation in sorted(operations_)[:3]:
1181
+ click.echo(_style(f" - {operation}", fg="yellow"))
1182
+ extra_count = len(operations_) - 3
1183
+ if extra_count > 0:
1184
+ click.echo(_style(f" + {extra_count} more", fg="yellow"))
1185
+
1186
+ if isinstance(operations, dict):
1187
+ for status_code, ops in operations.items():
1157
1188
  status_text = "Unauthorized" if status_code == 401 else "Forbidden"
1158
- count = len(operations)
1189
+ count = len(ops)
1159
1190
  suffix = "" if count == 1 else "s"
1160
- click.echo(
1161
- _style(
1162
- f"{status_code} {status_text} ({count} operation{suffix}):",
1163
- fg="yellow",
1164
- )
1165
- )
1166
- # Show first few API operations
1167
- for endpoint in sorted(operations)[:3]:
1168
- click.echo(_style(f" - {endpoint}", fg="yellow"))
1169
- if len(operations) > 3:
1170
- click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
1171
- click.echo()
1172
- click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
1173
- click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
1174
- click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
1175
- click.echo(_style("to provide authentication credentials", fg="yellow"))
1176
- click.echo()
1191
+ click.echo(_style(f"{status_code} {status_text} ({count} operation{suffix}):", fg="yellow"))
1177
1192
 
1178
- if self.warnings.only_4xx_responses:
1179
- count = len(self.warnings.only_4xx_responses)
1180
- suffix = "" if count == 1 else "s"
1181
- click.echo(
1182
- _style(
1183
- f"Schemathesis configuration: {count} operation{suffix} returned only 4xx responses during unit tests\n",
1184
- fg="yellow",
1185
- )
1186
- )
1193
+ _print_up_to_three(ops)
1194
+ else:
1195
+ _print_up_to_three(operations)
1187
1196
 
1188
- for endpoint in sorted(self.warnings.only_4xx_responses)[:3]:
1189
- click.echo(_style(f" - {endpoint}", fg="yellow"))
1190
- if len(self.warnings.only_4xx_responses) > 3:
1191
- click.echo(_style(f" + {len(self.warnings.only_4xx_responses) - 3} more", fg="yellow"))
1197
+ if tips:
1192
1198
  click.echo()
1193
1199
 
1194
- click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
1195
- click.echo(_style("Check base URL or adjust data generation settings", fg="yellow"))
1196
- click.echo()
1200
+ for tip in tips:
1201
+ click.echo(_style(tip, fg="yellow"))
1197
1202
 
1198
- def display_experiments(self) -> None:
1199
- display_section_name("EXPERIMENTS")
1203
+ click.echo()
1200
1204
 
1205
+ def display_warnings(self) -> None:
1206
+ display_section_name("WARNINGS")
1201
1207
  click.echo()
1202
- for experiment in sorted(GLOBAL_EXPERIMENTS.enabled, key=lambda e: e.name):
1203
- click.echo(_style(f"🧪 {experiment.name}: ", bold=True), nl=False)
1204
- click.echo(_style(experiment.description))
1205
- click.echo(_style(f" Feedback: {experiment.discussion_url}"))
1206
- click.echo()
1208
+ if self.warnings.missing_auth:
1209
+ self._display_warning_block(
1210
+ title="Missing authentication",
1211
+ operations=self.warnings.missing_auth,
1212
+ operation_suffix=" returned authentication errors",
1213
+ tips=["💡 Use --auth or -H to provide authentication credentials"],
1214
+ )
1207
1215
 
1208
- click.echo(
1209
- _style(
1210
- "Your feedback is crucial for experimental features. "
1211
- "Please visit the provided URL(s) to share your thoughts.",
1212
- dim=True,
1216
+ if self.warnings.missing_test_data:
1217
+ self._display_warning_block(
1218
+ title="Missing test data",
1219
+ operations=self.warnings.missing_test_data,
1220
+ operation_suffix=" repeatedly returned 404 Not Found, preventing tests from reaching your API's core logic",
1221
+ tips=[
1222
+ "💡 Provide realistic parameter values in your config file so tests can access existing resources",
1223
+ ],
1224
+ )
1225
+
1226
+ if self.warnings.validation_mismatch:
1227
+ self._display_warning_block(
1228
+ title="Schema validation mismatch",
1229
+ operations=self.warnings.validation_mismatch,
1230
+ operation_suffix=" mostly rejected generated data due to validation errors, indicating schema constraints don't match API validation",
1231
+ tips=["💡 Check your schema constraints - API validation may be stricter than documented"],
1213
1232
  )
1214
- )
1215
- click.echo()
1216
1233
 
1217
1234
  def display_stateful_failures(self, ctx: ExecutionContext) -> None:
1218
1235
  display_section_name("Stateful tests")
@@ -1255,7 +1272,7 @@ class OutputHandler(EventHandler):
1255
1272
  click.echo(f"\n{indent}<EMPTY>")
1256
1273
  else:
1257
1274
  try:
1258
- payload = prepare_response_payload(response.text, config=ctx.output_config)
1275
+ payload = prepare_response_payload(response.text, config=ctx.config.output)
1259
1276
  click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
1260
1277
  except UnicodeDecodeError:
1261
1278
  click.echo(f"\n{indent}<BINARY>")
@@ -1410,24 +1427,27 @@ class OutputHandler(EventHandler):
1410
1427
  display_section_name(message, fg=color)
1411
1428
 
1412
1429
  def display_reports(self) -> None:
1413
- if self.report_config is not None:
1414
- reports = [
1415
- (format.value.upper(), self.report_config.get_path(format).name)
1416
- for format in ReportFormat
1417
- if format in self.report_config.formats
1418
- ]
1419
-
1430
+ reports = self.config.reports
1431
+ if reports.vcr.enabled or reports.har.enabled or reports.junit.enabled:
1420
1432
  click.echo(_style("Reports:", bold=True))
1421
- for report_type, path in reports:
1422
- click.echo(_style(f" - {report_type}: {path}"))
1433
+ for format, report in (
1434
+ (ReportFormat.JUNIT, reports.junit),
1435
+ (ReportFormat.VCR, reports.vcr),
1436
+ (ReportFormat.HAR, reports.har),
1437
+ ):
1438
+ if report.enabled:
1439
+ path = reports.get_path(format)
1440
+ click.echo(_style(f" - {format.value.upper()}: {path}"))
1423
1441
  click.echo()
1424
1442
 
1425
1443
  def display_seed(self) -> None:
1426
1444
  click.echo(_style("Seed: ", bold=True), nl=False)
1427
- if self.seed is None:
1445
+ # Deterministic mode can be applied to a subset of tests, but we only care if it is enabled everywhere
1446
+ # If not everywhere, then the seed matter and should be displayed
1447
+ if self.config.seed is None or self.config.generation.deterministic:
1428
1448
  click.echo("not used in the deterministic mode")
1429
1449
  else:
1430
- click.echo(str(self.seed))
1450
+ click.echo(str(self.config.seed))
1431
1451
  click.echo()
1432
1452
 
1433
1453
  def _on_engine_finished(self, ctx: ExecutionContext, event: events.EngineFinished) -> None:
@@ -1438,9 +1458,13 @@ class OutputHandler(EventHandler):
1438
1458
  if self.errors:
1439
1459
  display_section_name("ERRORS")
1440
1460
  errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
1441
- for error in errors:
1442
- display_section_name(error.label, "_", fg="red")
1443
- click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1461
+ for label, group_errors in groupby(errors, key=lambda r: r.label):
1462
+ display_section_name(label, "_", fg="red")
1463
+ _errors = list(group_errors)
1464
+ for idx, error in enumerate(_errors, 1):
1465
+ click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1466
+ if idx < len(_errors):
1467
+ click.echo()
1444
1468
  click.echo(
1445
1469
  _style(
1446
1470
  f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
@@ -1448,10 +1472,8 @@ class OutputHandler(EventHandler):
1448
1472
  )
1449
1473
  )
1450
1474
  display_failures(ctx)
1451
- if self.warnings.missing_auth or self.warnings.only_4xx_responses:
1475
+ if not self.warnings.is_empty:
1452
1476
  self.display_warnings()
1453
- if GLOBAL_EXPERIMENTS.enabled:
1454
- self.display_experiments()
1455
1477
  if ctx.statistic.extraction_failures:
1456
1478
  self.display_stateful_failures(ctx)
1457
1479
  display_section_name("SUMMARY")
@@ -1468,21 +1490,39 @@ class OutputHandler(EventHandler):
1468
1490
  if self.errors:
1469
1491
  self.display_errors_summary()
1470
1492
 
1471
- if self.warnings.missing_auth or self.warnings.only_4xx_responses:
1493
+ if not self.warnings.is_empty:
1472
1494
  click.echo(_style("Warnings:", bold=True))
1473
1495
 
1474
1496
  if self.warnings.missing_auth:
1475
1497
  affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
1476
- click.echo(_style(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
1498
+ suffix = "" if affected == 1 else "s"
1499
+ click.echo(
1500
+ _style(
1501
+ f" ⚠️ Missing authentication: {bold(str(affected))} operation{suffix} returned only 401/403 responses",
1502
+ fg="yellow",
1503
+ )
1504
+ )
1477
1505
 
1478
- if self.warnings.only_4xx_responses:
1479
- count = len(self.warnings.only_4xx_responses)
1506
+ if self.warnings.missing_test_data:
1507
+ count = len(self.warnings.missing_test_data)
1480
1508
  suffix = "" if count == 1 else "s"
1481
1509
  click.echo(
1482
- _style(f" ⚠️ Schemathesis configuration: {bold(str(count))}", fg="yellow"),
1483
- nl=False,
1510
+ _style(
1511
+ f" ⚠️ Missing valid test data: {bold(str(count))} operation{suffix} repeatedly returned 404 responses",
1512
+ fg="yellow",
1513
+ )
1514
+ )
1515
+
1516
+ if self.warnings.validation_mismatch:
1517
+ count = len(self.warnings.validation_mismatch)
1518
+ suffix = "" if count == 1 else "s"
1519
+ click.echo(
1520
+ _style(
1521
+ f" ⚠️ Schema validation mismatch: {bold(str(count))} operation{suffix} mostly rejected generated data",
1522
+ fg="yellow",
1523
+ )
1484
1524
  )
1485
- click.echo(_style(f" operation{suffix} returned only 4xx responses during unit tests", fg="yellow"))
1525
+
1486
1526
  click.echo()
1487
1527
 
1488
1528
  if ctx.summary_lines:
@@ -1495,12 +1535,6 @@ class OutputHandler(EventHandler):
1495
1535
  self.display_final_line(ctx, event)
1496
1536
 
1497
1537
 
1498
- TOO_MANY_RESPONSES_WARNING_TEMPLATE = (
1499
- "Most of the responses from {} have a {} status code. Did you specify proper API credentials?"
1500
- )
1501
- TOO_MANY_RESPONSES_THRESHOLD = 0.9
1502
-
1503
-
1504
1538
  @dataclass
1505
1539
  class StatusCodeStatistic:
1506
1540
  """Statistics about HTTP status codes in a scenario."""
@@ -1516,15 +1550,55 @@ class StatusCodeStatistic:
1516
1550
  return 0.0
1517
1551
  return self.counts.get(status_code, 0) / self.total
1518
1552
 
1519
- def should_warn_about_only_4xx(self) -> bool:
1520
- """Check if an operation should be warned about (only 4xx responses, excluding auth)."""
1553
+ def _get_4xx_breakdown(self) -> tuple[int, int, int]:
1554
+ """Get breakdown of 4xx responses: (404_count, other_4xx_count, total_4xx_count)."""
1555
+ count_404 = self.counts.get(404, 0)
1556
+ count_other_4xx = sum(
1557
+ count for code, count in self.counts.items() if 400 <= code < 500 and code not in {401, 403, 404}
1558
+ )
1559
+ total_4xx = count_404 + count_other_4xx
1560
+ return count_404, count_other_4xx, total_4xx
1561
+
1562
+ def _is_only_4xx_responses(self) -> bool:
1563
+ """Check if all responses are 4xx (excluding 5xx)."""
1564
+ return all(400 <= code < 500 for code in self.counts.keys() if code not in {500})
1565
+
1566
+ def _can_warn_about_4xx(self) -> bool:
1567
+ """Check basic conditions for 4xx warnings."""
1521
1568
  if self.total == 0:
1522
1569
  return False
1523
- # Don't duplicate auth warnings
1524
- if set(self.counts.keys()) <= {401, 403}:
1570
+ # Skip if only auth errors
1571
+ if set(self.counts.keys()) <= {401, 403, 500}:
1572
+ return False
1573
+ return self._is_only_4xx_responses()
1574
+
1575
+ def should_warn_about_missing_test_data(self) -> bool:
1576
+ """Check if an operation should be warned about missing test data (significant 404 responses)."""
1577
+ if not self._can_warn_about_4xx():
1578
+ return False
1579
+
1580
+ count_404, _, total_4xx = self._get_4xx_breakdown()
1581
+
1582
+ if total_4xx == 0:
1583
+ return False
1584
+
1585
+ return (count_404 / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1586
+
1587
+ def should_warn_about_validation_mismatch(self) -> bool:
1588
+ """Check if an operation should be warned about validation mismatch (significant 400/422 responses)."""
1589
+ if not self._can_warn_about_4xx():
1590
+ return False
1591
+
1592
+ _, count_other_4xx, total_4xx = self._get_4xx_breakdown()
1593
+
1594
+ if total_4xx == 0:
1525
1595
  return False
1526
- # At this point we know we only have 4xx responses
1527
- return True
1596
+
1597
+ return (count_other_4xx / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1598
+
1599
+
1600
+ AUTH_ERRORS_THRESHOLD = 0.9
1601
+ OTHER_CLIENT_ERRORS_THRESHOLD = 0.1
1528
1602
 
1529
1603
 
1530
1604
  def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
@@ -6,42 +6,28 @@ supporting both GraphQL and OpenAPI specifications.
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import os
9
10
  import warnings
10
- from dataclasses import dataclass
11
11
  from typing import TYPE_CHECKING, Any, Callable
12
12
 
13
13
  from schemathesis import graphql, openapi
14
- from schemathesis.core import NOT_SET, NotSet
14
+ from schemathesis.config import ProjectConfig
15
15
  from schemathesis.core.errors import LoaderError, LoaderErrorKind
16
16
  from schemathesis.core.fs import file_exists
17
- from schemathesis.core.output import OutputConfig
18
- from schemathesis.generation import GenerationConfig
19
17
 
20
18
  if TYPE_CHECKING:
21
- from schemathesis.engine.config import NetworkConfig
22
19
  from schemathesis.schemas import BaseSchema
23
20
 
24
- Loader = Callable[["AutodetectConfig"], "BaseSchema"]
21
+ Loader = Callable[["ProjectConfig"], "BaseSchema"]
25
22
 
26
23
 
27
- @dataclass
28
- class AutodetectConfig:
29
- location: str
30
- network: NetworkConfig
31
- wait_for_schema: float | None
32
- base_url: str | None | NotSet = NOT_SET
33
- rate_limit: str | None | NotSet = NOT_SET
34
- generation: GenerationConfig | NotSet = NOT_SET
35
- output: OutputConfig | NotSet = NOT_SET
36
-
37
-
38
- def load_schema(config: AutodetectConfig) -> BaseSchema:
24
+ def load_schema(location: str, config: ProjectConfig) -> BaseSchema:
39
25
  """Load API schema automatically based on the provided configuration."""
40
- if is_probably_graphql(config.location):
26
+ if is_probably_graphql(location):
41
27
  # Try GraphQL first, then fallback to Open API
42
- return _try_load_schema(config, graphql, openapi)
28
+ return _try_load_schema(location, config, graphql, openapi)
43
29
  # Try Open API first, then fallback to GraphQL
44
- return _try_load_schema(config, openapi, graphql)
30
+ return _try_load_schema(location, config, openapi, graphql)
45
31
 
46
32
 
47
33
  def should_try_more(exc: LoaderError) -> bool:
@@ -60,27 +46,28 @@ def should_try_more(exc: LoaderError) -> bool:
60
46
  )
61
47
 
62
48
 
63
- def detect_loader(schema_or_location: str | dict[str, Any], module: Any) -> Callable:
49
+ def detect_loader(location: str, module: Any) -> Callable:
64
50
  """Detect API schema loader."""
65
- if isinstance(schema_or_location, str):
66
- if file_exists(schema_or_location):
67
- return module.from_path # type: ignore
68
- return module.from_url # type: ignore
69
- raise NotImplementedError
51
+ if file_exists(location):
52
+ return module.from_path # type: ignore
53
+ return module.from_url # type: ignore
70
54
 
71
55
 
72
- def _try_load_schema(config: AutodetectConfig, first_module: Any, second_module: Any) -> BaseSchema:
56
+ def _try_load_schema(location: str, config: ProjectConfig, first_module: Any, second_module: Any) -> BaseSchema:
73
57
  """Try to load schema with fallback option."""
74
58
  from urllib3.exceptions import InsecureRequestWarning
75
59
 
76
60
  with warnings.catch_warnings():
77
61
  warnings.simplefilter("ignore", InsecureRequestWarning)
78
62
  try:
79
- return _load_schema(config, first_module)
63
+ return _load_schema(location, config, first_module)
80
64
  except LoaderError as exc:
65
+ # If this was the OpenAPI loader on an explicit OpenAPI file, don't fallback
66
+ if first_module is openapi and is_openapi_file(location):
67
+ raise exc
81
68
  if should_try_more(exc):
82
69
  try:
83
- return _load_schema(config, second_module)
70
+ return _load_schema(location, config, second_module)
84
71
  except Exception as second_exc:
85
72
  if is_specific_exception(second_exc):
86
73
  raise second_exc
@@ -88,26 +75,23 @@ def _try_load_schema(config: AutodetectConfig, first_module: Any, second_module:
88
75
  raise exc
89
76
 
90
77
 
91
- def _load_schema(config: AutodetectConfig, module: Any) -> BaseSchema:
78
+ def _load_schema(location: str, config: ProjectConfig, module: Any) -> BaseSchema:
92
79
  """Unified schema loader for both GraphQL and OpenAPI."""
93
- loader = detect_loader(config.location, module)
80
+ loader = detect_loader(location, module)
94
81
 
95
82
  kwargs: dict = {}
96
83
  if loader is module.from_url:
97
84
  if config.wait_for_schema is not None:
98
85
  kwargs["wait_for_schema"] = config.wait_for_schema
99
- kwargs["verify"] = config.network.tls_verify
100
- if config.network.cert:
101
- kwargs["cert"] = config.network.cert
102
- if config.network.auth:
103
- kwargs["auth"] = config.network.auth
104
-
105
- return loader(config.location, **kwargs).configure(
106
- base_url=config.base_url,
107
- rate_limit=config.rate_limit,
108
- output=config.output,
109
- generation=config.generation,
110
- )
86
+ kwargs["verify"] = config.tls_verify
87
+ request_cert = config.request_cert_for()
88
+ if request_cert:
89
+ kwargs["cert"] = request_cert
90
+ auth = config.auth_for()
91
+ if auth is not None:
92
+ kwargs["auth"] = auth
93
+
94
+ return loader(location, config=config._parent, **kwargs)
111
95
 
112
96
 
113
97
  def is_specific_exception(exc: Exception) -> bool:
@@ -120,10 +104,11 @@ def is_specific_exception(exc: Exception) -> bool:
120
104
  )
121
105
 
122
106
 
123
- def is_probably_graphql(schema_or_location: str | dict[str, Any]) -> bool:
107
+ def is_probably_graphql(location: str) -> bool:
124
108
  """Detect whether it is likely that the given location is a GraphQL endpoint."""
125
- if isinstance(schema_or_location, str):
126
- return schema_or_location.endswith(("/graphql", "/graphql/", ".graphql", ".gql"))
127
- return "__schema" in schema_or_location or (
128
- "data" in schema_or_location and "__schema" in schema_or_location["data"]
129
- )
109
+ return location.endswith(("/graphql", "/graphql/", ".graphql", ".gql"))
110
+
111
+
112
+ def is_openapi_file(location: str) -> bool:
113
+ name = os.path.basename(location).lower()
114
+ return any(name == f"{base}{ext}" for base in ("openapi", "swagger") for ext in (".json", ".yaml", ".yml"))