schemathesis 4.0.0a11__py3-none-any.whl → 4.0.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. schemathesis/__init__.py +35 -27
  2. schemathesis/auths.py +85 -54
  3. schemathesis/checks.py +65 -36
  4. schemathesis/cli/commands/run/__init__.py +32 -27
  5. schemathesis/cli/commands/run/context.py +6 -1
  6. schemathesis/cli/commands/run/events.py +7 -1
  7. schemathesis/cli/commands/run/executor.py +12 -7
  8. schemathesis/cli/commands/run/handlers/output.py +188 -80
  9. schemathesis/cli/commands/run/validation.py +21 -6
  10. schemathesis/cli/constants.py +1 -1
  11. schemathesis/config/__init__.py +2 -1
  12. schemathesis/config/_generation.py +12 -13
  13. schemathesis/config/_operations.py +14 -0
  14. schemathesis/config/_phases.py +41 -5
  15. schemathesis/config/_projects.py +33 -1
  16. schemathesis/config/_report.py +6 -2
  17. schemathesis/config/_warnings.py +25 -0
  18. schemathesis/config/schema.json +49 -1
  19. schemathesis/core/errors.py +15 -19
  20. schemathesis/core/transport.py +117 -2
  21. schemathesis/engine/context.py +1 -0
  22. schemathesis/engine/errors.py +61 -2
  23. schemathesis/engine/events.py +10 -2
  24. schemathesis/engine/phases/probes.py +3 -0
  25. schemathesis/engine/phases/stateful/__init__.py +2 -1
  26. schemathesis/engine/phases/stateful/_executor.py +38 -5
  27. schemathesis/engine/phases/stateful/context.py +2 -2
  28. schemathesis/engine/phases/unit/_executor.py +36 -7
  29. schemathesis/generation/__init__.py +0 -3
  30. schemathesis/generation/case.py +153 -28
  31. schemathesis/generation/coverage.py +1 -1
  32. schemathesis/generation/hypothesis/builder.py +43 -19
  33. schemathesis/generation/metrics.py +93 -0
  34. schemathesis/generation/modes.py +0 -8
  35. schemathesis/generation/overrides.py +11 -27
  36. schemathesis/generation/stateful/__init__.py +17 -0
  37. schemathesis/generation/stateful/state_machine.py +32 -108
  38. schemathesis/graphql/loaders.py +152 -8
  39. schemathesis/hooks.py +63 -39
  40. schemathesis/openapi/checks.py +82 -20
  41. schemathesis/openapi/generation/filters.py +9 -2
  42. schemathesis/openapi/loaders.py +134 -8
  43. schemathesis/pytest/lazy.py +4 -31
  44. schemathesis/pytest/loaders.py +24 -0
  45. schemathesis/pytest/plugin.py +38 -6
  46. schemathesis/schemas.py +161 -94
  47. schemathesis/specs/graphql/scalars.py +37 -3
  48. schemathesis/specs/graphql/schemas.py +18 -9
  49. schemathesis/specs/openapi/_hypothesis.py +53 -34
  50. schemathesis/specs/openapi/checks.py +111 -47
  51. schemathesis/specs/openapi/expressions/nodes.py +1 -1
  52. schemathesis/specs/openapi/formats.py +30 -3
  53. schemathesis/specs/openapi/media_types.py +44 -1
  54. schemathesis/specs/openapi/negative/__init__.py +5 -3
  55. schemathesis/specs/openapi/negative/mutations.py +2 -2
  56. schemathesis/specs/openapi/parameters.py +0 -3
  57. schemathesis/specs/openapi/schemas.py +14 -93
  58. schemathesis/specs/openapi/stateful/__init__.py +2 -1
  59. schemathesis/specs/openapi/stateful/links.py +1 -63
  60. schemathesis/transport/__init__.py +54 -16
  61. schemathesis/transport/prepare.py +31 -7
  62. schemathesis/transport/requests.py +21 -9
  63. schemathesis/transport/serialization.py +0 -4
  64. schemathesis/transport/wsgi.py +15 -8
  65. {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/METADATA +45 -87
  66. {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/RECORD +69 -71
  67. schemathesis/contrib/__init__.py +0 -9
  68. schemathesis/contrib/openapi/__init__.py +0 -9
  69. schemathesis/contrib/openapi/fill_missing_examples.py +0 -20
  70. schemathesis/generation/targets.py +0 -69
  71. {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/WHEEL +0 -0
  72. {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/entry_points.txt +0 -0
  73. {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/licenses/LICENSE +0 -0
@@ -4,6 +4,7 @@ import os
4
4
  import textwrap
5
5
  import time
6
6
  from dataclasses import dataclass, field
7
+ from itertools import groupby
7
8
  from json.decoder import JSONDecodeError
8
9
  from types import GeneratorType
9
10
  from typing import TYPE_CHECKING, Any, Generator, Iterable
@@ -15,7 +16,7 @@ from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
15
16
  from schemathesis.cli.commands.run.handlers.base import EventHandler
16
17
  from schemathesis.cli.constants import ISSUE_TRACKER_URL
17
18
  from schemathesis.cli.core import get_terminal_width
18
- from schemathesis.config import ProjectConfig, ReportFormat
19
+ from schemathesis.config import ProjectConfig, ReportFormat, SchemathesisWarning
19
20
  from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
20
21
  from schemathesis.core.failures import MessageBlock, Severity, format_failures
21
22
  from schemathesis.core.output import prepare_response_payload
@@ -324,7 +325,13 @@ class ProbingProgressManager:
324
325
  @dataclass
325
326
  class WarningData:
326
327
  missing_auth: dict[int, set[str]] = field(default_factory=dict)
327
- only_4xx_responses: set[str] = field(default_factory=set) # operations that only returned 4xx
328
+ missing_test_data: set[str] = field(default_factory=set)
329
+ # operations that only returned 4xx
330
+ validation_mismatch: set[str] = field(default_factory=set)
331
+
332
+ @property
333
+ def is_empty(self) -> bool:
334
+ return not bool(self.missing_auth or self.missing_test_data or self.validation_mismatch)
328
335
 
329
336
 
330
337
  @dataclass
@@ -792,7 +799,7 @@ class OutputHandler(EventHandler):
792
799
  elif isinstance(event, events.ScenarioStarted):
793
800
  self._on_scenario_started(event)
794
801
  elif isinstance(event, events.ScenarioFinished):
795
- self._on_scenario_finished(event)
802
+ self._on_scenario_finished(ctx, event)
796
803
  if isinstance(event, events.EngineFinished):
797
804
  self._on_engine_finished(ctx, event)
798
805
  elif isinstance(event, events.Interrupted):
@@ -1014,7 +1021,7 @@ class OutputHandler(EventHandler):
1014
1021
  assert self.unit_tests_manager is not None
1015
1022
  self.unit_tests_manager.start_operation(event.label)
1016
1023
 
1017
- def _on_scenario_finished(self, event: events.ScenarioFinished) -> None:
1024
+ def _on_scenario_finished(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1018
1025
  if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
1019
1026
  assert self.unit_tests_manager is not None
1020
1027
  if event.label:
@@ -1023,7 +1030,7 @@ class OutputHandler(EventHandler):
1023
1030
  self.unit_tests_manager.update_stats(event.status)
1024
1031
  if event.status == Status.SKIP and event.skip_reason is not None:
1025
1032
  self.skip_reasons.append(event.skip_reason)
1026
- self._check_warnings(event)
1033
+ self._check_warnings(ctx, event)
1027
1034
  elif (
1028
1035
  event.phase == PhaseName.STATEFUL_TESTING
1029
1036
  and not event.is_final
@@ -1032,16 +1039,24 @@ class OutputHandler(EventHandler):
1032
1039
  assert self.stateful_tests_manager is not None
1033
1040
  links_seen = {case.transition.id for case in event.recorder.cases.values() if case.transition is not None}
1034
1041
  self.stateful_tests_manager.update(links_seen, event.status)
1042
+ self._check_stateful_warnings(ctx, event)
1035
1043
 
1036
- def _check_warnings(self, event: events.ScenarioFinished) -> None:
1044
+ def _check_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1037
1045
  statistic = aggregate_status_codes(event.recorder.interactions.values())
1038
1046
 
1039
1047
  if statistic.total == 0:
1040
1048
  return
1041
1049
 
1042
- for status_code in (401, 403):
1043
- if statistic.ratio_for(status_code) >= TOO_MANY_RESPONSES_THRESHOLD:
1044
- self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
1050
+ assert ctx.find_operation_by_label is not None
1051
+ assert event.label is not None
1052
+ operation = ctx.find_operation_by_label(event.label)
1053
+
1054
+ warnings = self.config.warnings_for(operation=operation)
1055
+
1056
+ if SchemathesisWarning.MISSING_AUTH in warnings:
1057
+ for status_code in (401, 403):
1058
+ if statistic.ratio_for(status_code) >= AUTH_ERRORS_THRESHOLD:
1059
+ self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
1045
1060
 
1046
1061
  # Warn if all positive test cases got 4xx in return and no failure was found
1047
1062
  def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
@@ -1062,11 +1077,31 @@ class OutputHandler(EventHandler):
1062
1077
 
1063
1078
  if (
1064
1079
  event.status == Status.SUCCESS
1065
- and GenerationMode.POSITIVE in self.config.generation_for(operation=None, phase=event.phase.name).modes
1080
+ and (
1081
+ SchemathesisWarning.MISSING_TEST_DATA in warnings or SchemathesisWarning.VALIDATION_MISMATCH in warnings
1082
+ )
1083
+ and GenerationMode.POSITIVE in self.config.generation_for(operation=operation, phase=event.phase.name).modes
1066
1084
  and all_positive_are_rejected(event.recorder)
1067
- and statistic.should_warn_about_only_4xx()
1068
1085
  ):
1069
- self.warnings.only_4xx_responses.add(event.recorder.label)
1086
+ if SchemathesisWarning.MISSING_TEST_DATA in warnings and statistic.should_warn_about_missing_test_data():
1087
+ self.warnings.missing_test_data.add(event.recorder.label)
1088
+ if (
1089
+ SchemathesisWarning.VALIDATION_MISMATCH in warnings
1090
+ and statistic.should_warn_about_validation_mismatch()
1091
+ ):
1092
+ self.warnings.validation_mismatch.add(event.recorder.label)
1093
+
1094
+ def _check_stateful_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
1095
+ # If stateful testing had successful responses for API operations that were marked with "missing_test_data"
1096
+ # warnings, then remove them from warnings
1097
+ for key, node in event.recorder.cases.items():
1098
+ if not self.warnings.missing_test_data:
1099
+ break
1100
+ if node.value.operation.label in self.warnings.missing_test_data and key in event.recorder.interactions:
1101
+ response = event.recorder.interactions[key].response
1102
+ if response is not None and response.status_code < 300:
1103
+ self.warnings.missing_test_data.remove(node.value.operation.label)
1104
+ continue
1070
1105
 
1071
1106
  def _on_interrupted(self, event: events.Interrupted) -> None:
1072
1107
  from rich.padding import Padding
@@ -1137,60 +1172,77 @@ class OutputHandler(EventHandler):
1137
1172
 
1138
1173
  raise click.Abort
1139
1174
 
1140
- def display_warnings(self) -> None:
1141
- display_section_name("WARNINGS")
1142
- click.echo()
1143
- if self.warnings.missing_auth:
1144
- total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
1145
- suffix = "" if total == 1 else "s"
1146
- click.echo(
1147
- _style(
1148
- f"Missing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
1149
- fg="yellow",
1150
- )
1175
+ def _display_warning_block(
1176
+ self, title: str, operations: set[str] | dict, tips: list[str], operation_suffix: str = ""
1177
+ ) -> None:
1178
+ if isinstance(operations, dict):
1179
+ total = sum(len(ops) for ops in operations.values())
1180
+ else:
1181
+ total = len(operations)
1182
+
1183
+ suffix = "" if total == 1 else "s"
1184
+ click.echo(
1185
+ _style(
1186
+ f"{title}: {total} operation{suffix}{operation_suffix}\n",
1187
+ fg="yellow",
1151
1188
  )
1189
+ )
1190
+
1191
+ # Print up to 3 endpoints, then "+N more"
1192
+ def _print_up_to_three(operations_: list[str] | set[str]) -> None:
1193
+ for operation in sorted(operations_)[:3]:
1194
+ click.echo(_style(f" - {operation}", fg="yellow"))
1195
+ extra_count = len(operations_) - 3
1196
+ if extra_count > 0:
1197
+ click.echo(_style(f" + {extra_count} more", fg="yellow"))
1152
1198
 
1153
- for status_code, operations in self.warnings.missing_auth.items():
1199
+ if isinstance(operations, dict):
1200
+ for status_code, ops in operations.items():
1154
1201
  status_text = "Unauthorized" if status_code == 401 else "Forbidden"
1155
- count = len(operations)
1202
+ count = len(ops)
1156
1203
  suffix = "" if count == 1 else "s"
1157
- click.echo(
1158
- _style(
1159
- f"{status_code} {status_text} ({count} operation{suffix}):",
1160
- fg="yellow",
1161
- )
1162
- )
1163
- # Show first few API operations
1164
- for endpoint in sorted(operations)[:3]:
1165
- click.echo(_style(f" - {endpoint}", fg="yellow"))
1166
- if len(operations) > 3:
1167
- click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
1168
- click.echo()
1169
- click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
1170
- click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
1171
- click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
1172
- click.echo(_style("to provide authentication credentials", fg="yellow"))
1204
+ click.echo(_style(f"{status_code} {status_text} ({count} operation{suffix}):", fg="yellow"))
1205
+
1206
+ _print_up_to_three(ops)
1207
+ else:
1208
+ _print_up_to_three(operations)
1209
+
1210
+ if tips:
1173
1211
  click.echo()
1174
1212
 
1175
- if self.warnings.only_4xx_responses:
1176
- count = len(self.warnings.only_4xx_responses)
1177
- suffix = "" if count == 1 else "s"
1178
- click.echo(
1179
- _style(
1180
- f"Schemathesis configuration: {count} operation{suffix} returned only 4xx responses during unit tests\n",
1181
- fg="yellow",
1182
- )
1213
+ for tip in tips:
1214
+ click.echo(_style(tip, fg="yellow"))
1215
+
1216
+ click.echo()
1217
+
1218
+ def display_warnings(self) -> None:
1219
+ display_section_name("WARNINGS")
1220
+ click.echo()
1221
+ if self.warnings.missing_auth:
1222
+ self._display_warning_block(
1223
+ title="Missing authentication",
1224
+ operations=self.warnings.missing_auth,
1225
+ operation_suffix=" returned authentication errors",
1226
+ tips=["💡 Use --auth or -H to provide authentication credentials"],
1183
1227
  )
1184
1228
 
1185
- for endpoint in sorted(self.warnings.only_4xx_responses)[:3]:
1186
- click.echo(_style(f" - {endpoint}", fg="yellow"))
1187
- if len(self.warnings.only_4xx_responses) > 3:
1188
- click.echo(_style(f" + {len(self.warnings.only_4xx_responses) - 3} more", fg="yellow"))
1189
- click.echo()
1229
+ if self.warnings.missing_test_data:
1230
+ self._display_warning_block(
1231
+ title="Missing test data",
1232
+ operations=self.warnings.missing_test_data,
1233
+ operation_suffix=" repeatedly returned 404 Not Found, preventing tests from reaching your API's core logic",
1234
+ tips=[
1235
+ "💡 Provide realistic parameter values in your config file so tests can access existing resources",
1236
+ ],
1237
+ )
1190
1238
 
1191
- click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
1192
- click.echo(_style("Check base URL or adjust data generation settings", fg="yellow"))
1193
- click.echo()
1239
+ if self.warnings.validation_mismatch:
1240
+ self._display_warning_block(
1241
+ title="Schema validation mismatch",
1242
+ operations=self.warnings.validation_mismatch,
1243
+ operation_suffix=" mostly rejected generated data due to validation errors, indicating schema constraints don't match API validation",
1244
+ tips=["💡 Check your schema constraints - API validation may be stricter than documented"],
1245
+ )
1194
1246
 
1195
1247
  def display_stateful_failures(self, ctx: ExecutionContext) -> None:
1196
1248
  display_section_name("Stateful tests")
@@ -1419,9 +1471,13 @@ class OutputHandler(EventHandler):
1419
1471
  if self.errors:
1420
1472
  display_section_name("ERRORS")
1421
1473
  errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
1422
- for error in errors:
1423
- display_section_name(error.label, "_", fg="red")
1424
- click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1474
+ for label, group_errors in groupby(errors, key=lambda r: r.label):
1475
+ display_section_name(label, "_", fg="red")
1476
+ _errors = list(group_errors)
1477
+ for idx, error in enumerate(_errors, 1):
1478
+ click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
1479
+ if idx < len(_errors):
1480
+ click.echo()
1425
1481
  click.echo(
1426
1482
  _style(
1427
1483
  f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
@@ -1429,7 +1485,7 @@ class OutputHandler(EventHandler):
1429
1485
  )
1430
1486
  )
1431
1487
  display_failures(ctx)
1432
- if self.warnings.missing_auth or self.warnings.only_4xx_responses:
1488
+ if not self.warnings.is_empty:
1433
1489
  self.display_warnings()
1434
1490
  if ctx.statistic.extraction_failures:
1435
1491
  self.display_stateful_failures(ctx)
@@ -1447,21 +1503,39 @@ class OutputHandler(EventHandler):
1447
1503
  if self.errors:
1448
1504
  self.display_errors_summary()
1449
1505
 
1450
- if self.warnings.missing_auth or self.warnings.only_4xx_responses:
1506
+ if not self.warnings.is_empty:
1451
1507
  click.echo(_style("Warnings:", bold=True))
1452
1508
 
1453
1509
  if self.warnings.missing_auth:
1454
1510
  affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
1455
- click.echo(_style(f" ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
1511
+ suffix = "" if affected == 1 else "s"
1512
+ click.echo(
1513
+ _style(
1514
+ f" ⚠️ Missing authentication: {bold(str(affected))} operation{suffix} returned only 401/403 responses",
1515
+ fg="yellow",
1516
+ )
1517
+ )
1518
+
1519
+ if self.warnings.missing_test_data:
1520
+ count = len(self.warnings.missing_test_data)
1521
+ suffix = "" if count == 1 else "s"
1522
+ click.echo(
1523
+ _style(
1524
+ f" ⚠️ Missing valid test data: {bold(str(count))} operation{suffix} repeatedly returned 404 responses",
1525
+ fg="yellow",
1526
+ )
1527
+ )
1456
1528
 
1457
- if self.warnings.only_4xx_responses:
1458
- count = len(self.warnings.only_4xx_responses)
1529
+ if self.warnings.validation_mismatch:
1530
+ count = len(self.warnings.validation_mismatch)
1459
1531
  suffix = "" if count == 1 else "s"
1460
1532
  click.echo(
1461
- _style(f" ⚠️ Schemathesis configuration: {bold(str(count))}", fg="yellow"),
1462
- nl=False,
1533
+ _style(
1534
+ f" ⚠️ Schema validation mismatch: {bold(str(count))} operation{suffix} mostly rejected generated data",
1535
+ fg="yellow",
1536
+ )
1463
1537
  )
1464
- click.echo(_style(f" operation{suffix} returned only 4xx responses during unit tests", fg="yellow"))
1538
+
1465
1539
  click.echo()
1466
1540
 
1467
1541
  if ctx.summary_lines:
@@ -1474,12 +1548,6 @@ class OutputHandler(EventHandler):
1474
1548
  self.display_final_line(ctx, event)
1475
1549
 
1476
1550
 
1477
- TOO_MANY_RESPONSES_WARNING_TEMPLATE = (
1478
- "Most of the responses from {} have a {} status code. Did you specify proper API credentials?"
1479
- )
1480
- TOO_MANY_RESPONSES_THRESHOLD = 0.9
1481
-
1482
-
1483
1551
  @dataclass
1484
1552
  class StatusCodeStatistic:
1485
1553
  """Statistics about HTTP status codes in a scenario."""
@@ -1495,15 +1563,55 @@ class StatusCodeStatistic:
1495
1563
  return 0.0
1496
1564
  return self.counts.get(status_code, 0) / self.total
1497
1565
 
1498
- def should_warn_about_only_4xx(self) -> bool:
1499
- """Check if an operation should be warned about (only 4xx responses, excluding auth)."""
1566
+ def _get_4xx_breakdown(self) -> tuple[int, int, int]:
1567
+ """Get breakdown of 4xx responses: (404_count, other_4xx_count, total_4xx_count)."""
1568
+ count_404 = self.counts.get(404, 0)
1569
+ count_other_4xx = sum(
1570
+ count for code, count in self.counts.items() if 400 <= code < 500 and code not in {401, 403, 404}
1571
+ )
1572
+ total_4xx = count_404 + count_other_4xx
1573
+ return count_404, count_other_4xx, total_4xx
1574
+
1575
+ def _is_only_4xx_responses(self) -> bool:
1576
+ """Check if all responses are 4xx (excluding 5xx)."""
1577
+ return all(400 <= code < 500 for code in self.counts.keys() if code not in {500})
1578
+
1579
+ def _can_warn_about_4xx(self) -> bool:
1580
+ """Check basic conditions for 4xx warnings."""
1500
1581
  if self.total == 0:
1501
1582
  return False
1502
- # Don't duplicate auth warnings
1503
- if set(self.counts.keys()) <= {401, 403}:
1583
+ # Skip if only auth errors
1584
+ if set(self.counts.keys()) <= {401, 403, 500}:
1585
+ return False
1586
+ return self._is_only_4xx_responses()
1587
+
1588
+ def should_warn_about_missing_test_data(self) -> bool:
1589
+ """Check if an operation should be warned about missing test data (significant 404 responses)."""
1590
+ if not self._can_warn_about_4xx():
1591
+ return False
1592
+
1593
+ count_404, _, total_4xx = self._get_4xx_breakdown()
1594
+
1595
+ if total_4xx == 0:
1504
1596
  return False
1505
- # At this point we know we only have 4xx responses
1506
- return True
1597
+
1598
+ return (count_404 / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1599
+
1600
+ def should_warn_about_validation_mismatch(self) -> bool:
1601
+ """Check if an operation should be warned about validation mismatch (significant 400/422 responses)."""
1602
+ if not self._can_warn_about_4xx():
1603
+ return False
1604
+
1605
+ _, count_other_4xx, total_4xx = self._get_4xx_breakdown()
1606
+
1607
+ if total_4xx == 0:
1608
+ return False
1609
+
1610
+ return (count_other_4xx / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
1611
+
1612
+
1613
+ AUTH_ERRORS_THRESHOLD = 0.9
1614
+ OTHER_CLIENT_ERRORS_THRESHOLD = 0.1
1507
1615
 
1508
1616
 
1509
1617
  def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
@@ -10,13 +10,14 @@ from urllib.parse import urlparse
10
10
 
11
11
  import click
12
12
 
13
- from schemathesis.config import ReportFormat, get_workers_count
13
+ from schemathesis.cli.ext.options import CsvEnumChoice
14
+ from schemathesis.config import ReportFormat, SchemathesisWarning, get_workers_count
14
15
  from schemathesis.core import errors, rate_limit, string_to_boolean
15
16
  from schemathesis.core.fs import file_exists
16
17
  from schemathesis.core.validation import has_invalid_characters, is_latin_1_encodable
17
18
  from schemathesis.filters import expression_to_filter_function
18
19
  from schemathesis.generation import GenerationMode
19
- from schemathesis.generation.targets import TargetFunction
20
+ from schemathesis.generation.metrics import MetricFunction
20
21
 
21
22
  INVALID_DERANDOMIZE_MESSAGE = (
22
23
  "`--generation-deterministic` implies no database, so passing `--generation-database` too is invalid."
@@ -200,16 +201,16 @@ def reduce_list(
200
201
 
201
202
  def convert_maximize(
202
203
  ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]
203
- ) -> list[TargetFunction]:
204
- from schemathesis.generation.targets import TARGETS
204
+ ) -> list[MetricFunction]:
205
+ from schemathesis.generation.metrics import METRICS
205
206
 
206
207
  names: list[str] = reduce(operator.iadd, value, [])
207
- return TARGETS.get_by_names(names)
208
+ return METRICS.get_by_names(names)
208
209
 
209
210
 
210
211
  def convert_generation_mode(ctx: click.core.Context, param: click.core.Parameter, value: str) -> list[GenerationMode]:
211
212
  if value == "all":
212
- return GenerationMode.all()
213
+ return list(GenerationMode)
213
214
  return [GenerationMode(value)]
214
215
 
215
216
 
@@ -229,3 +230,17 @@ def convert_workers(ctx: click.core.Context, param: click.core.Parameter, value:
229
230
  if value == "auto":
230
231
  return get_workers_count()
231
232
  return int(value)
233
+
234
+
235
+ WARNINGS_CHOICE = CsvEnumChoice(SchemathesisWarning)
236
+
237
+
238
+ def validate_warnings(
239
+ ctx: click.core.Context, param: click.core.Parameter, value: str | None
240
+ ) -> bool | None | list[SchemathesisWarning]:
241
+ if value is None:
242
+ return None
243
+ boolean = string_to_boolean(value)
244
+ if isinstance(boolean, bool):
245
+ return boolean
246
+ return WARNINGS_CHOICE.convert(value, param, ctx) # type: ignore[return-value]
@@ -5,4 +5,4 @@ ISSUE_TRACKER_URL = (
5
5
  "https://github.com/schemathesis/schemathesis/issues/new?"
6
6
  "labels=Status%3A%20Needs%20Triage%2C+Type%3A+Bug&template=bug_report.md&title=%5BBUG%5D"
7
7
  )
8
- EXTENSIONS_DOCUMENTATION_URL = "https://schemathesis.readthedocs.io/en/stable/extending.html"
8
+ EXTENSIONS_DOCUMENTATION_URL = "https://schemathesis.readthedocs.io/en/latest/guides/extending/"
@@ -20,7 +20,7 @@ from schemathesis.config._generation import GenerationConfig
20
20
  from schemathesis.config._health_check import HealthCheck
21
21
  from schemathesis.config._output import OutputConfig, SanitizationConfig, TruncationConfig
22
22
  from schemathesis.config._phases import CoveragePhaseConfig, PhaseConfig, PhasesConfig, StatefulPhaseConfig
23
- from schemathesis.config._projects import ProjectConfig, ProjectsConfig, get_workers_count
23
+ from schemathesis.config._projects import ProjectConfig, ProjectsConfig, SchemathesisWarning, get_workers_count
24
24
  from schemathesis.config._report import DEFAULT_REPORT_DIRECTORY, ReportConfig, ReportFormat, ReportsConfig
25
25
 
26
26
  __all__ = [
@@ -47,6 +47,7 @@ __all__ = [
47
47
  "ProjectsConfig",
48
48
  "ProjectConfig",
49
49
  "get_workers_count",
50
+ "SchemathesisWarning",
50
51
  ]
51
52
 
52
53
 
@@ -7,7 +7,7 @@ from schemathesis.config._diff_base import DiffBase
7
7
  from schemathesis.generation.modes import GenerationMode
8
8
 
9
9
  if TYPE_CHECKING:
10
- from schemathesis.generation.targets import TargetFunction
10
+ from schemathesis.generation.metrics import MetricFunction
11
11
 
12
12
 
13
13
  @dataclass(repr=False)
@@ -20,7 +20,7 @@ class GenerationConfig(DiffBase):
20
20
  allow_x00: bool
21
21
  # Generate strings using the given codec
22
22
  codec: str | None
23
- maximize: list[TargetFunction]
23
+ maximize: list[MetricFunction]
24
24
  # Whether to generate security parameters
25
25
  with_security_parameters: bool
26
26
  # Allowing using `null` for optional arguments in GraphQL queries
@@ -53,7 +53,7 @@ class GenerationConfig(DiffBase):
53
53
  deterministic: bool = False,
54
54
  allow_x00: bool = True,
55
55
  codec: str | None = "utf-8",
56
- maximize: list[TargetFunction] | None = None,
56
+ maximize: list[MetricFunction] | None = None,
57
57
  with_security_parameters: bool = True,
58
58
  graphql_allow_null: bool = True,
59
59
  database: str | None = None,
@@ -62,8 +62,7 @@ class GenerationConfig(DiffBase):
62
62
  ) -> None:
63
63
  from schemathesis.generation import GenerationMode
64
64
 
65
- # TODO: Switch to `all` by default.
66
- self.modes = modes or [GenerationMode.POSITIVE]
65
+ self.modes = modes or list(GenerationMode)
67
66
  self.max_examples = max_examples
68
67
  self.no_shrink = no_shrink
69
68
  self.deterministic = deterministic
@@ -80,7 +79,7 @@ class GenerationConfig(DiffBase):
80
79
  def from_dict(cls, data: dict[str, Any]) -> GenerationConfig:
81
80
  mode_raw = data.get("mode")
82
81
  if mode_raw == "all":
83
- modes = GenerationMode.all()
82
+ modes = list(GenerationMode)
84
83
  elif mode_raw is not None:
85
84
  modes = [GenerationMode(mode_raw)]
86
85
  else:
@@ -110,7 +109,7 @@ class GenerationConfig(DiffBase):
110
109
  deterministic: bool | None = None,
111
110
  allow_x00: bool = True,
112
111
  codec: str | None = None,
113
- maximize: list[TargetFunction] | None = None,
112
+ maximize: list[MetricFunction] | None = None,
114
113
  with_security_parameters: bool | None = None,
115
114
  graphql_allow_null: bool = True,
116
115
  database: str | None = None,
@@ -138,13 +137,13 @@ class GenerationConfig(DiffBase):
138
137
  self.exclude_header_characters = exclude_header_characters
139
138
 
140
139
 
141
- def _get_maximize(value: Any) -> list[TargetFunction]:
142
- from schemathesis.generation.targets import TARGETS
140
+ def _get_maximize(value: Any) -> list[MetricFunction]:
141
+ from schemathesis.generation.metrics import METRICS
143
142
 
144
143
  if isinstance(value, list):
145
- targets = value
144
+ metrics = value
146
145
  elif isinstance(value, str):
147
- targets = [value]
146
+ metrics = [value]
148
147
  else:
149
- targets = []
150
- return TARGETS.get_by_names(targets)
148
+ metrics = []
149
+ return METRICS.get_by_names(metrics)
@@ -14,6 +14,7 @@ from schemathesis.config._generation import GenerationConfig
14
14
  from schemathesis.config._parameters import load_parameters
15
15
  from schemathesis.config._phases import PhasesConfig
16
16
  from schemathesis.config._rate_limit import build_limiter
17
+ from schemathesis.config._warnings import SchemathesisWarning, resolve_warnings
17
18
  from schemathesis.core.errors import IncorrectUsage
18
19
  from schemathesis.filters import FilterSet, HasAPIOperation, expression_to_filter_function, is_deprecated
19
20
 
@@ -207,6 +208,7 @@ class OperationConfig(DiffBase):
207
208
  request_cert: str | None
208
209
  request_cert_key: str | None
209
210
  parameters: dict[str, Any]
211
+ warnings: list[SchemathesisWarning] | None
210
212
  auth: AuthConfig
211
213
  checks: ChecksConfig
212
214
  phases: PhasesConfig
@@ -225,6 +227,7 @@ class OperationConfig(DiffBase):
225
227
  "request_cert",
226
228
  "request_cert_key",
227
229
  "parameters",
230
+ "warnings",
228
231
  "auth",
229
232
  "checks",
230
233
  "phases",
@@ -245,6 +248,7 @@ class OperationConfig(DiffBase):
245
248
  request_cert: str | None = None,
246
249
  request_cert_key: str | None = None,
247
250
  parameters: dict[str, Any] | None = None,
251
+ warnings: bool | list[SchemathesisWarning] | None = None,
248
252
  auth: AuthConfig | None = None,
249
253
  checks: ChecksConfig | None = None,
250
254
  phases: PhasesConfig | None = None,
@@ -265,6 +269,7 @@ class OperationConfig(DiffBase):
265
269
  self.request_cert = request_cert
266
270
  self.request_cert_key = request_cert_key
267
271
  self.parameters = parameters or {}
272
+ self._set_warnings(warnings)
268
273
  self.auth = auth or AuthConfig()
269
274
  self.checks = checks or ChecksConfig()
270
275
  self.phases = phases or PhasesConfig()
@@ -306,8 +311,17 @@ class OperationConfig(DiffBase):
306
311
  request_cert=resolve(data.get("request-cert")),
307
312
  request_cert_key=resolve(data.get("request-cert-key")),
308
313
  parameters=load_parameters(data),
314
+ warnings=resolve_warnings(data.get("warnings")),
309
315
  auth=AuthConfig.from_dict(data.get("auth", {})),
310
316
  checks=ChecksConfig.from_dict(data.get("checks", {})),
311
317
  phases=PhasesConfig.from_dict(data.get("phases", {})),
312
318
  generation=GenerationConfig.from_dict(data.get("generation", {})),
313
319
  )
320
+
321
+ def _set_warnings(self, warnings: bool | list[SchemathesisWarning] | None) -> None:
322
+ if warnings is False:
323
+ self.warnings = []
324
+ elif warnings is True:
325
+ self.warnings = list(SchemathesisWarning)
326
+ else:
327
+ self.warnings = warnings