schemathesis 4.0.0a11__py3-none-any.whl → 4.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/__init__.py +35 -27
- schemathesis/auths.py +85 -54
- schemathesis/checks.py +65 -36
- schemathesis/cli/commands/run/__init__.py +32 -27
- schemathesis/cli/commands/run/context.py +6 -1
- schemathesis/cli/commands/run/events.py +7 -1
- schemathesis/cli/commands/run/executor.py +12 -7
- schemathesis/cli/commands/run/handlers/output.py +188 -80
- schemathesis/cli/commands/run/validation.py +21 -6
- schemathesis/cli/constants.py +1 -1
- schemathesis/config/__init__.py +2 -1
- schemathesis/config/_generation.py +12 -13
- schemathesis/config/_operations.py +14 -0
- schemathesis/config/_phases.py +41 -5
- schemathesis/config/_projects.py +33 -1
- schemathesis/config/_report.py +6 -2
- schemathesis/config/_warnings.py +25 -0
- schemathesis/config/schema.json +49 -1
- schemathesis/core/errors.py +15 -19
- schemathesis/core/transport.py +117 -2
- schemathesis/engine/context.py +1 -0
- schemathesis/engine/errors.py +61 -2
- schemathesis/engine/events.py +10 -2
- schemathesis/engine/phases/probes.py +3 -0
- schemathesis/engine/phases/stateful/__init__.py +2 -1
- schemathesis/engine/phases/stateful/_executor.py +38 -5
- schemathesis/engine/phases/stateful/context.py +2 -2
- schemathesis/engine/phases/unit/_executor.py +36 -7
- schemathesis/generation/__init__.py +0 -3
- schemathesis/generation/case.py +153 -28
- schemathesis/generation/coverage.py +1 -1
- schemathesis/generation/hypothesis/builder.py +43 -19
- schemathesis/generation/metrics.py +93 -0
- schemathesis/generation/modes.py +0 -8
- schemathesis/generation/overrides.py +11 -27
- schemathesis/generation/stateful/__init__.py +17 -0
- schemathesis/generation/stateful/state_machine.py +32 -108
- schemathesis/graphql/loaders.py +152 -8
- schemathesis/hooks.py +63 -39
- schemathesis/openapi/checks.py +82 -20
- schemathesis/openapi/generation/filters.py +9 -2
- schemathesis/openapi/loaders.py +134 -8
- schemathesis/pytest/lazy.py +4 -31
- schemathesis/pytest/loaders.py +24 -0
- schemathesis/pytest/plugin.py +38 -6
- schemathesis/schemas.py +161 -94
- schemathesis/specs/graphql/scalars.py +37 -3
- schemathesis/specs/graphql/schemas.py +18 -9
- schemathesis/specs/openapi/_hypothesis.py +53 -34
- schemathesis/specs/openapi/checks.py +111 -47
- schemathesis/specs/openapi/expressions/nodes.py +1 -1
- schemathesis/specs/openapi/formats.py +30 -3
- schemathesis/specs/openapi/media_types.py +44 -1
- schemathesis/specs/openapi/negative/__init__.py +5 -3
- schemathesis/specs/openapi/negative/mutations.py +2 -2
- schemathesis/specs/openapi/parameters.py +0 -3
- schemathesis/specs/openapi/schemas.py +14 -93
- schemathesis/specs/openapi/stateful/__init__.py +2 -1
- schemathesis/specs/openapi/stateful/links.py +1 -63
- schemathesis/transport/__init__.py +54 -16
- schemathesis/transport/prepare.py +31 -7
- schemathesis/transport/requests.py +21 -9
- schemathesis/transport/serialization.py +0 -4
- schemathesis/transport/wsgi.py +15 -8
- {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/METADATA +45 -87
- {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/RECORD +69 -71
- schemathesis/contrib/__init__.py +0 -9
- schemathesis/contrib/openapi/__init__.py +0 -9
- schemathesis/contrib/openapi/fill_missing_examples.py +0 -20
- schemathesis/generation/targets.py +0 -69
- {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a11.dist-info → schemathesis-4.0.0b1.dist-info}/licenses/LICENSE +0 -0
@@ -4,6 +4,7 @@ import os
|
|
4
4
|
import textwrap
|
5
5
|
import time
|
6
6
|
from dataclasses import dataclass, field
|
7
|
+
from itertools import groupby
|
7
8
|
from json.decoder import JSONDecodeError
|
8
9
|
from types import GeneratorType
|
9
10
|
from typing import TYPE_CHECKING, Any, Generator, Iterable
|
@@ -15,7 +16,7 @@ from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
|
|
15
16
|
from schemathesis.cli.commands.run.handlers.base import EventHandler
|
16
17
|
from schemathesis.cli.constants import ISSUE_TRACKER_URL
|
17
18
|
from schemathesis.cli.core import get_terminal_width
|
18
|
-
from schemathesis.config import ProjectConfig, ReportFormat
|
19
|
+
from schemathesis.config import ProjectConfig, ReportFormat, SchemathesisWarning
|
19
20
|
from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
|
20
21
|
from schemathesis.core.failures import MessageBlock, Severity, format_failures
|
21
22
|
from schemathesis.core.output import prepare_response_payload
|
@@ -324,7 +325,13 @@ class ProbingProgressManager:
|
|
324
325
|
@dataclass
|
325
326
|
class WarningData:
|
326
327
|
missing_auth: dict[int, set[str]] = field(default_factory=dict)
|
327
|
-
|
328
|
+
missing_test_data: set[str] = field(default_factory=set)
|
329
|
+
# operations that only returned 4xx
|
330
|
+
validation_mismatch: set[str] = field(default_factory=set)
|
331
|
+
|
332
|
+
@property
|
333
|
+
def is_empty(self) -> bool:
|
334
|
+
return not bool(self.missing_auth or self.missing_test_data or self.validation_mismatch)
|
328
335
|
|
329
336
|
|
330
337
|
@dataclass
|
@@ -792,7 +799,7 @@ class OutputHandler(EventHandler):
|
|
792
799
|
elif isinstance(event, events.ScenarioStarted):
|
793
800
|
self._on_scenario_started(event)
|
794
801
|
elif isinstance(event, events.ScenarioFinished):
|
795
|
-
self._on_scenario_finished(event)
|
802
|
+
self._on_scenario_finished(ctx, event)
|
796
803
|
if isinstance(event, events.EngineFinished):
|
797
804
|
self._on_engine_finished(ctx, event)
|
798
805
|
elif isinstance(event, events.Interrupted):
|
@@ -1014,7 +1021,7 @@ class OutputHandler(EventHandler):
|
|
1014
1021
|
assert self.unit_tests_manager is not None
|
1015
1022
|
self.unit_tests_manager.start_operation(event.label)
|
1016
1023
|
|
1017
|
-
def _on_scenario_finished(self, event: events.ScenarioFinished) -> None:
|
1024
|
+
def _on_scenario_finished(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
|
1018
1025
|
if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
|
1019
1026
|
assert self.unit_tests_manager is not None
|
1020
1027
|
if event.label:
|
@@ -1023,7 +1030,7 @@ class OutputHandler(EventHandler):
|
|
1023
1030
|
self.unit_tests_manager.update_stats(event.status)
|
1024
1031
|
if event.status == Status.SKIP and event.skip_reason is not None:
|
1025
1032
|
self.skip_reasons.append(event.skip_reason)
|
1026
|
-
self._check_warnings(event)
|
1033
|
+
self._check_warnings(ctx, event)
|
1027
1034
|
elif (
|
1028
1035
|
event.phase == PhaseName.STATEFUL_TESTING
|
1029
1036
|
and not event.is_final
|
@@ -1032,16 +1039,24 @@ class OutputHandler(EventHandler):
|
|
1032
1039
|
assert self.stateful_tests_manager is not None
|
1033
1040
|
links_seen = {case.transition.id for case in event.recorder.cases.values() if case.transition is not None}
|
1034
1041
|
self.stateful_tests_manager.update(links_seen, event.status)
|
1042
|
+
self._check_stateful_warnings(ctx, event)
|
1035
1043
|
|
1036
|
-
def _check_warnings(self, event: events.ScenarioFinished) -> None:
|
1044
|
+
def _check_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
|
1037
1045
|
statistic = aggregate_status_codes(event.recorder.interactions.values())
|
1038
1046
|
|
1039
1047
|
if statistic.total == 0:
|
1040
1048
|
return
|
1041
1049
|
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1050
|
+
assert ctx.find_operation_by_label is not None
|
1051
|
+
assert event.label is not None
|
1052
|
+
operation = ctx.find_operation_by_label(event.label)
|
1053
|
+
|
1054
|
+
warnings = self.config.warnings_for(operation=operation)
|
1055
|
+
|
1056
|
+
if SchemathesisWarning.MISSING_AUTH in warnings:
|
1057
|
+
for status_code in (401, 403):
|
1058
|
+
if statistic.ratio_for(status_code) >= AUTH_ERRORS_THRESHOLD:
|
1059
|
+
self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
|
1045
1060
|
|
1046
1061
|
# Warn if all positive test cases got 4xx in return and no failure was found
|
1047
1062
|
def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
|
@@ -1062,11 +1077,31 @@ class OutputHandler(EventHandler):
|
|
1062
1077
|
|
1063
1078
|
if (
|
1064
1079
|
event.status == Status.SUCCESS
|
1065
|
-
and
|
1080
|
+
and (
|
1081
|
+
SchemathesisWarning.MISSING_TEST_DATA in warnings or SchemathesisWarning.VALIDATION_MISMATCH in warnings
|
1082
|
+
)
|
1083
|
+
and GenerationMode.POSITIVE in self.config.generation_for(operation=operation, phase=event.phase.name).modes
|
1066
1084
|
and all_positive_are_rejected(event.recorder)
|
1067
|
-
and statistic.should_warn_about_only_4xx()
|
1068
1085
|
):
|
1069
|
-
|
1086
|
+
if SchemathesisWarning.MISSING_TEST_DATA in warnings and statistic.should_warn_about_missing_test_data():
|
1087
|
+
self.warnings.missing_test_data.add(event.recorder.label)
|
1088
|
+
if (
|
1089
|
+
SchemathesisWarning.VALIDATION_MISMATCH in warnings
|
1090
|
+
and statistic.should_warn_about_validation_mismatch()
|
1091
|
+
):
|
1092
|
+
self.warnings.validation_mismatch.add(event.recorder.label)
|
1093
|
+
|
1094
|
+
def _check_stateful_warnings(self, ctx: ExecutionContext, event: events.ScenarioFinished) -> None:
|
1095
|
+
# If stateful testing had successful responses for API operations that were marked with "missing_test_data"
|
1096
|
+
# warnings, then remove them from warnings
|
1097
|
+
for key, node in event.recorder.cases.items():
|
1098
|
+
if not self.warnings.missing_test_data:
|
1099
|
+
break
|
1100
|
+
if node.value.operation.label in self.warnings.missing_test_data and key in event.recorder.interactions:
|
1101
|
+
response = event.recorder.interactions[key].response
|
1102
|
+
if response is not None and response.status_code < 300:
|
1103
|
+
self.warnings.missing_test_data.remove(node.value.operation.label)
|
1104
|
+
continue
|
1070
1105
|
|
1071
1106
|
def _on_interrupted(self, event: events.Interrupted) -> None:
|
1072
1107
|
from rich.padding import Padding
|
@@ -1137,60 +1172,77 @@ class OutputHandler(EventHandler):
|
|
1137
1172
|
|
1138
1173
|
raise click.Abort
|
1139
1174
|
|
1140
|
-
def
|
1141
|
-
|
1142
|
-
|
1143
|
-
if
|
1144
|
-
total = sum(len(
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1175
|
+
def _display_warning_block(
|
1176
|
+
self, title: str, operations: set[str] | dict, tips: list[str], operation_suffix: str = ""
|
1177
|
+
) -> None:
|
1178
|
+
if isinstance(operations, dict):
|
1179
|
+
total = sum(len(ops) for ops in operations.values())
|
1180
|
+
else:
|
1181
|
+
total = len(operations)
|
1182
|
+
|
1183
|
+
suffix = "" if total == 1 else "s"
|
1184
|
+
click.echo(
|
1185
|
+
_style(
|
1186
|
+
f"{title}: {total} operation{suffix}{operation_suffix}\n",
|
1187
|
+
fg="yellow",
|
1151
1188
|
)
|
1189
|
+
)
|
1190
|
+
|
1191
|
+
# Print up to 3 endpoints, then "+N more"
|
1192
|
+
def _print_up_to_three(operations_: list[str] | set[str]) -> None:
|
1193
|
+
for operation in sorted(operations_)[:3]:
|
1194
|
+
click.echo(_style(f" - {operation}", fg="yellow"))
|
1195
|
+
extra_count = len(operations_) - 3
|
1196
|
+
if extra_count > 0:
|
1197
|
+
click.echo(_style(f" + {extra_count} more", fg="yellow"))
|
1152
1198
|
|
1153
|
-
|
1199
|
+
if isinstance(operations, dict):
|
1200
|
+
for status_code, ops in operations.items():
|
1154
1201
|
status_text = "Unauthorized" if status_code == 401 else "Forbidden"
|
1155
|
-
count = len(
|
1202
|
+
count = len(ops)
|
1156
1203
|
suffix = "" if count == 1 else "s"
|
1157
|
-
click.echo(
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1164
|
-
for endpoint in sorted(operations)[:3]:
|
1165
|
-
click.echo(_style(f" - {endpoint}", fg="yellow"))
|
1166
|
-
if len(operations) > 3:
|
1167
|
-
click.echo(_style(f" + {len(operations) - 3} more", fg="yellow"))
|
1168
|
-
click.echo()
|
1169
|
-
click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
|
1170
|
-
click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
|
1171
|
-
click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
|
1172
|
-
click.echo(_style("to provide authentication credentials", fg="yellow"))
|
1204
|
+
click.echo(_style(f"{status_code} {status_text} ({count} operation{suffix}):", fg="yellow"))
|
1205
|
+
|
1206
|
+
_print_up_to_three(ops)
|
1207
|
+
else:
|
1208
|
+
_print_up_to_three(operations)
|
1209
|
+
|
1210
|
+
if tips:
|
1173
1211
|
click.echo()
|
1174
1212
|
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
|
1182
|
-
|
1213
|
+
for tip in tips:
|
1214
|
+
click.echo(_style(tip, fg="yellow"))
|
1215
|
+
|
1216
|
+
click.echo()
|
1217
|
+
|
1218
|
+
def display_warnings(self) -> None:
|
1219
|
+
display_section_name("WARNINGS")
|
1220
|
+
click.echo()
|
1221
|
+
if self.warnings.missing_auth:
|
1222
|
+
self._display_warning_block(
|
1223
|
+
title="Missing authentication",
|
1224
|
+
operations=self.warnings.missing_auth,
|
1225
|
+
operation_suffix=" returned authentication errors",
|
1226
|
+
tips=["💡 Use --auth or -H to provide authentication credentials"],
|
1183
1227
|
)
|
1184
1228
|
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
|
1189
|
-
|
1229
|
+
if self.warnings.missing_test_data:
|
1230
|
+
self._display_warning_block(
|
1231
|
+
title="Missing test data",
|
1232
|
+
operations=self.warnings.missing_test_data,
|
1233
|
+
operation_suffix=" repeatedly returned 404 Not Found, preventing tests from reaching your API's core logic",
|
1234
|
+
tips=[
|
1235
|
+
"💡 Provide realistic parameter values in your config file so tests can access existing resources",
|
1236
|
+
],
|
1237
|
+
)
|
1190
1238
|
|
1191
|
-
|
1192
|
-
|
1193
|
-
|
1239
|
+
if self.warnings.validation_mismatch:
|
1240
|
+
self._display_warning_block(
|
1241
|
+
title="Schema validation mismatch",
|
1242
|
+
operations=self.warnings.validation_mismatch,
|
1243
|
+
operation_suffix=" mostly rejected generated data due to validation errors, indicating schema constraints don't match API validation",
|
1244
|
+
tips=["💡 Check your schema constraints - API validation may be stricter than documented"],
|
1245
|
+
)
|
1194
1246
|
|
1195
1247
|
def display_stateful_failures(self, ctx: ExecutionContext) -> None:
|
1196
1248
|
display_section_name("Stateful tests")
|
@@ -1419,9 +1471,13 @@ class OutputHandler(EventHandler):
|
|
1419
1471
|
if self.errors:
|
1420
1472
|
display_section_name("ERRORS")
|
1421
1473
|
errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
|
1422
|
-
for
|
1423
|
-
display_section_name(
|
1424
|
-
|
1474
|
+
for label, group_errors in groupby(errors, key=lambda r: r.label):
|
1475
|
+
display_section_name(label, "_", fg="red")
|
1476
|
+
_errors = list(group_errors)
|
1477
|
+
for idx, error in enumerate(_errors, 1):
|
1478
|
+
click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
|
1479
|
+
if idx < len(_errors):
|
1480
|
+
click.echo()
|
1425
1481
|
click.echo(
|
1426
1482
|
_style(
|
1427
1483
|
f"\nNeed more help?\n Join our Discord server: {DISCORD_LINK}",
|
@@ -1429,7 +1485,7 @@ class OutputHandler(EventHandler):
|
|
1429
1485
|
)
|
1430
1486
|
)
|
1431
1487
|
display_failures(ctx)
|
1432
|
-
if
|
1488
|
+
if not self.warnings.is_empty:
|
1433
1489
|
self.display_warnings()
|
1434
1490
|
if ctx.statistic.extraction_failures:
|
1435
1491
|
self.display_stateful_failures(ctx)
|
@@ -1447,21 +1503,39 @@ class OutputHandler(EventHandler):
|
|
1447
1503
|
if self.errors:
|
1448
1504
|
self.display_errors_summary()
|
1449
1505
|
|
1450
|
-
if
|
1506
|
+
if not self.warnings.is_empty:
|
1451
1507
|
click.echo(_style("Warnings:", bold=True))
|
1452
1508
|
|
1453
1509
|
if self.warnings.missing_auth:
|
1454
1510
|
affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
|
1455
|
-
|
1511
|
+
suffix = "" if affected == 1 else "s"
|
1512
|
+
click.echo(
|
1513
|
+
_style(
|
1514
|
+
f" ⚠️ Missing authentication: {bold(str(affected))} operation{suffix} returned only 401/403 responses",
|
1515
|
+
fg="yellow",
|
1516
|
+
)
|
1517
|
+
)
|
1518
|
+
|
1519
|
+
if self.warnings.missing_test_data:
|
1520
|
+
count = len(self.warnings.missing_test_data)
|
1521
|
+
suffix = "" if count == 1 else "s"
|
1522
|
+
click.echo(
|
1523
|
+
_style(
|
1524
|
+
f" ⚠️ Missing valid test data: {bold(str(count))} operation{suffix} repeatedly returned 404 responses",
|
1525
|
+
fg="yellow",
|
1526
|
+
)
|
1527
|
+
)
|
1456
1528
|
|
1457
|
-
if self.warnings.
|
1458
|
-
count = len(self.warnings.
|
1529
|
+
if self.warnings.validation_mismatch:
|
1530
|
+
count = len(self.warnings.validation_mismatch)
|
1459
1531
|
suffix = "" if count == 1 else "s"
|
1460
1532
|
click.echo(
|
1461
|
-
_style(
|
1462
|
-
|
1533
|
+
_style(
|
1534
|
+
f" ⚠️ Schema validation mismatch: {bold(str(count))} operation{suffix} mostly rejected generated data",
|
1535
|
+
fg="yellow",
|
1536
|
+
)
|
1463
1537
|
)
|
1464
|
-
|
1538
|
+
|
1465
1539
|
click.echo()
|
1466
1540
|
|
1467
1541
|
if ctx.summary_lines:
|
@@ -1474,12 +1548,6 @@ class OutputHandler(EventHandler):
|
|
1474
1548
|
self.display_final_line(ctx, event)
|
1475
1549
|
|
1476
1550
|
|
1477
|
-
TOO_MANY_RESPONSES_WARNING_TEMPLATE = (
|
1478
|
-
"Most of the responses from {} have a {} status code. Did you specify proper API credentials?"
|
1479
|
-
)
|
1480
|
-
TOO_MANY_RESPONSES_THRESHOLD = 0.9
|
1481
|
-
|
1482
|
-
|
1483
1551
|
@dataclass
|
1484
1552
|
class StatusCodeStatistic:
|
1485
1553
|
"""Statistics about HTTP status codes in a scenario."""
|
@@ -1495,15 +1563,55 @@ class StatusCodeStatistic:
|
|
1495
1563
|
return 0.0
|
1496
1564
|
return self.counts.get(status_code, 0) / self.total
|
1497
1565
|
|
1498
|
-
def
|
1499
|
-
"""
|
1566
|
+
def _get_4xx_breakdown(self) -> tuple[int, int, int]:
|
1567
|
+
"""Get breakdown of 4xx responses: (404_count, other_4xx_count, total_4xx_count)."""
|
1568
|
+
count_404 = self.counts.get(404, 0)
|
1569
|
+
count_other_4xx = sum(
|
1570
|
+
count for code, count in self.counts.items() if 400 <= code < 500 and code not in {401, 403, 404}
|
1571
|
+
)
|
1572
|
+
total_4xx = count_404 + count_other_4xx
|
1573
|
+
return count_404, count_other_4xx, total_4xx
|
1574
|
+
|
1575
|
+
def _is_only_4xx_responses(self) -> bool:
|
1576
|
+
"""Check if all responses are 4xx (excluding 5xx)."""
|
1577
|
+
return all(400 <= code < 500 for code in self.counts.keys() if code not in {500})
|
1578
|
+
|
1579
|
+
def _can_warn_about_4xx(self) -> bool:
|
1580
|
+
"""Check basic conditions for 4xx warnings."""
|
1500
1581
|
if self.total == 0:
|
1501
1582
|
return False
|
1502
|
-
#
|
1503
|
-
if set(self.counts.keys()) <= {401, 403}:
|
1583
|
+
# Skip if only auth errors
|
1584
|
+
if set(self.counts.keys()) <= {401, 403, 500}:
|
1585
|
+
return False
|
1586
|
+
return self._is_only_4xx_responses()
|
1587
|
+
|
1588
|
+
def should_warn_about_missing_test_data(self) -> bool:
|
1589
|
+
"""Check if an operation should be warned about missing test data (significant 404 responses)."""
|
1590
|
+
if not self._can_warn_about_4xx():
|
1591
|
+
return False
|
1592
|
+
|
1593
|
+
count_404, _, total_4xx = self._get_4xx_breakdown()
|
1594
|
+
|
1595
|
+
if total_4xx == 0:
|
1504
1596
|
return False
|
1505
|
-
|
1506
|
-
return
|
1597
|
+
|
1598
|
+
return (count_404 / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
|
1599
|
+
|
1600
|
+
def should_warn_about_validation_mismatch(self) -> bool:
|
1601
|
+
"""Check if an operation should be warned about validation mismatch (significant 400/422 responses)."""
|
1602
|
+
if not self._can_warn_about_4xx():
|
1603
|
+
return False
|
1604
|
+
|
1605
|
+
_, count_other_4xx, total_4xx = self._get_4xx_breakdown()
|
1606
|
+
|
1607
|
+
if total_4xx == 0:
|
1608
|
+
return False
|
1609
|
+
|
1610
|
+
return (count_other_4xx / total_4xx) >= OTHER_CLIENT_ERRORS_THRESHOLD
|
1611
|
+
|
1612
|
+
|
1613
|
+
AUTH_ERRORS_THRESHOLD = 0.9
|
1614
|
+
OTHER_CLIENT_ERRORS_THRESHOLD = 0.1
|
1507
1615
|
|
1508
1616
|
|
1509
1617
|
def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
|
@@ -10,13 +10,14 @@ from urllib.parse import urlparse
|
|
10
10
|
|
11
11
|
import click
|
12
12
|
|
13
|
-
from schemathesis.
|
13
|
+
from schemathesis.cli.ext.options import CsvEnumChoice
|
14
|
+
from schemathesis.config import ReportFormat, SchemathesisWarning, get_workers_count
|
14
15
|
from schemathesis.core import errors, rate_limit, string_to_boolean
|
15
16
|
from schemathesis.core.fs import file_exists
|
16
17
|
from schemathesis.core.validation import has_invalid_characters, is_latin_1_encodable
|
17
18
|
from schemathesis.filters import expression_to_filter_function
|
18
19
|
from schemathesis.generation import GenerationMode
|
19
|
-
from schemathesis.generation.
|
20
|
+
from schemathesis.generation.metrics import MetricFunction
|
20
21
|
|
21
22
|
INVALID_DERANDOMIZE_MESSAGE = (
|
22
23
|
"`--generation-deterministic` implies no database, so passing `--generation-database` too is invalid."
|
@@ -200,16 +201,16 @@ def reduce_list(
|
|
200
201
|
|
201
202
|
def convert_maximize(
|
202
203
|
ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]
|
203
|
-
) -> list[
|
204
|
-
from schemathesis.generation.
|
204
|
+
) -> list[MetricFunction]:
|
205
|
+
from schemathesis.generation.metrics import METRICS
|
205
206
|
|
206
207
|
names: list[str] = reduce(operator.iadd, value, [])
|
207
|
-
return
|
208
|
+
return METRICS.get_by_names(names)
|
208
209
|
|
209
210
|
|
210
211
|
def convert_generation_mode(ctx: click.core.Context, param: click.core.Parameter, value: str) -> list[GenerationMode]:
|
211
212
|
if value == "all":
|
212
|
-
return GenerationMode
|
213
|
+
return list(GenerationMode)
|
213
214
|
return [GenerationMode(value)]
|
214
215
|
|
215
216
|
|
@@ -229,3 +230,17 @@ def convert_workers(ctx: click.core.Context, param: click.core.Parameter, value:
|
|
229
230
|
if value == "auto":
|
230
231
|
return get_workers_count()
|
231
232
|
return int(value)
|
233
|
+
|
234
|
+
|
235
|
+
WARNINGS_CHOICE = CsvEnumChoice(SchemathesisWarning)
|
236
|
+
|
237
|
+
|
238
|
+
def validate_warnings(
|
239
|
+
ctx: click.core.Context, param: click.core.Parameter, value: str | None
|
240
|
+
) -> bool | None | list[SchemathesisWarning]:
|
241
|
+
if value is None:
|
242
|
+
return None
|
243
|
+
boolean = string_to_boolean(value)
|
244
|
+
if isinstance(boolean, bool):
|
245
|
+
return boolean
|
246
|
+
return WARNINGS_CHOICE.convert(value, param, ctx) # type: ignore[return-value]
|
schemathesis/cli/constants.py
CHANGED
@@ -5,4 +5,4 @@ ISSUE_TRACKER_URL = (
|
|
5
5
|
"https://github.com/schemathesis/schemathesis/issues/new?"
|
6
6
|
"labels=Status%3A%20Needs%20Triage%2C+Type%3A+Bug&template=bug_report.md&title=%5BBUG%5D"
|
7
7
|
)
|
8
|
-
EXTENSIONS_DOCUMENTATION_URL = "https://schemathesis.readthedocs.io/en/
|
8
|
+
EXTENSIONS_DOCUMENTATION_URL = "https://schemathesis.readthedocs.io/en/latest/guides/extending/"
|
schemathesis/config/__init__.py
CHANGED
@@ -20,7 +20,7 @@ from schemathesis.config._generation import GenerationConfig
|
|
20
20
|
from schemathesis.config._health_check import HealthCheck
|
21
21
|
from schemathesis.config._output import OutputConfig, SanitizationConfig, TruncationConfig
|
22
22
|
from schemathesis.config._phases import CoveragePhaseConfig, PhaseConfig, PhasesConfig, StatefulPhaseConfig
|
23
|
-
from schemathesis.config._projects import ProjectConfig, ProjectsConfig, get_workers_count
|
23
|
+
from schemathesis.config._projects import ProjectConfig, ProjectsConfig, SchemathesisWarning, get_workers_count
|
24
24
|
from schemathesis.config._report import DEFAULT_REPORT_DIRECTORY, ReportConfig, ReportFormat, ReportsConfig
|
25
25
|
|
26
26
|
__all__ = [
|
@@ -47,6 +47,7 @@ __all__ = [
|
|
47
47
|
"ProjectsConfig",
|
48
48
|
"ProjectConfig",
|
49
49
|
"get_workers_count",
|
50
|
+
"SchemathesisWarning",
|
50
51
|
]
|
51
52
|
|
52
53
|
|
@@ -7,7 +7,7 @@ from schemathesis.config._diff_base import DiffBase
|
|
7
7
|
from schemathesis.generation.modes import GenerationMode
|
8
8
|
|
9
9
|
if TYPE_CHECKING:
|
10
|
-
from schemathesis.generation.
|
10
|
+
from schemathesis.generation.metrics import MetricFunction
|
11
11
|
|
12
12
|
|
13
13
|
@dataclass(repr=False)
|
@@ -20,7 +20,7 @@ class GenerationConfig(DiffBase):
|
|
20
20
|
allow_x00: bool
|
21
21
|
# Generate strings using the given codec
|
22
22
|
codec: str | None
|
23
|
-
maximize: list[
|
23
|
+
maximize: list[MetricFunction]
|
24
24
|
# Whether to generate security parameters
|
25
25
|
with_security_parameters: bool
|
26
26
|
# Allowing using `null` for optional arguments in GraphQL queries
|
@@ -53,7 +53,7 @@ class GenerationConfig(DiffBase):
|
|
53
53
|
deterministic: bool = False,
|
54
54
|
allow_x00: bool = True,
|
55
55
|
codec: str | None = "utf-8",
|
56
|
-
maximize: list[
|
56
|
+
maximize: list[MetricFunction] | None = None,
|
57
57
|
with_security_parameters: bool = True,
|
58
58
|
graphql_allow_null: bool = True,
|
59
59
|
database: str | None = None,
|
@@ -62,8 +62,7 @@ class GenerationConfig(DiffBase):
|
|
62
62
|
) -> None:
|
63
63
|
from schemathesis.generation import GenerationMode
|
64
64
|
|
65
|
-
|
66
|
-
self.modes = modes or [GenerationMode.POSITIVE]
|
65
|
+
self.modes = modes or list(GenerationMode)
|
67
66
|
self.max_examples = max_examples
|
68
67
|
self.no_shrink = no_shrink
|
69
68
|
self.deterministic = deterministic
|
@@ -80,7 +79,7 @@ class GenerationConfig(DiffBase):
|
|
80
79
|
def from_dict(cls, data: dict[str, Any]) -> GenerationConfig:
|
81
80
|
mode_raw = data.get("mode")
|
82
81
|
if mode_raw == "all":
|
83
|
-
modes = GenerationMode
|
82
|
+
modes = list(GenerationMode)
|
84
83
|
elif mode_raw is not None:
|
85
84
|
modes = [GenerationMode(mode_raw)]
|
86
85
|
else:
|
@@ -110,7 +109,7 @@ class GenerationConfig(DiffBase):
|
|
110
109
|
deterministic: bool | None = None,
|
111
110
|
allow_x00: bool = True,
|
112
111
|
codec: str | None = None,
|
113
|
-
maximize: list[
|
112
|
+
maximize: list[MetricFunction] | None = None,
|
114
113
|
with_security_parameters: bool | None = None,
|
115
114
|
graphql_allow_null: bool = True,
|
116
115
|
database: str | None = None,
|
@@ -138,13 +137,13 @@ class GenerationConfig(DiffBase):
|
|
138
137
|
self.exclude_header_characters = exclude_header_characters
|
139
138
|
|
140
139
|
|
141
|
-
def _get_maximize(value: Any) -> list[
|
142
|
-
from schemathesis.generation.
|
140
|
+
def _get_maximize(value: Any) -> list[MetricFunction]:
|
141
|
+
from schemathesis.generation.metrics import METRICS
|
143
142
|
|
144
143
|
if isinstance(value, list):
|
145
|
-
|
144
|
+
metrics = value
|
146
145
|
elif isinstance(value, str):
|
147
|
-
|
146
|
+
metrics = [value]
|
148
147
|
else:
|
149
|
-
|
150
|
-
return
|
148
|
+
metrics = []
|
149
|
+
return METRICS.get_by_names(metrics)
|
@@ -14,6 +14,7 @@ from schemathesis.config._generation import GenerationConfig
|
|
14
14
|
from schemathesis.config._parameters import load_parameters
|
15
15
|
from schemathesis.config._phases import PhasesConfig
|
16
16
|
from schemathesis.config._rate_limit import build_limiter
|
17
|
+
from schemathesis.config._warnings import SchemathesisWarning, resolve_warnings
|
17
18
|
from schemathesis.core.errors import IncorrectUsage
|
18
19
|
from schemathesis.filters import FilterSet, HasAPIOperation, expression_to_filter_function, is_deprecated
|
19
20
|
|
@@ -207,6 +208,7 @@ class OperationConfig(DiffBase):
|
|
207
208
|
request_cert: str | None
|
208
209
|
request_cert_key: str | None
|
209
210
|
parameters: dict[str, Any]
|
211
|
+
warnings: list[SchemathesisWarning] | None
|
210
212
|
auth: AuthConfig
|
211
213
|
checks: ChecksConfig
|
212
214
|
phases: PhasesConfig
|
@@ -225,6 +227,7 @@ class OperationConfig(DiffBase):
|
|
225
227
|
"request_cert",
|
226
228
|
"request_cert_key",
|
227
229
|
"parameters",
|
230
|
+
"warnings",
|
228
231
|
"auth",
|
229
232
|
"checks",
|
230
233
|
"phases",
|
@@ -245,6 +248,7 @@ class OperationConfig(DiffBase):
|
|
245
248
|
request_cert: str | None = None,
|
246
249
|
request_cert_key: str | None = None,
|
247
250
|
parameters: dict[str, Any] | None = None,
|
251
|
+
warnings: bool | list[SchemathesisWarning] | None = None,
|
248
252
|
auth: AuthConfig | None = None,
|
249
253
|
checks: ChecksConfig | None = None,
|
250
254
|
phases: PhasesConfig | None = None,
|
@@ -265,6 +269,7 @@ class OperationConfig(DiffBase):
|
|
265
269
|
self.request_cert = request_cert
|
266
270
|
self.request_cert_key = request_cert_key
|
267
271
|
self.parameters = parameters or {}
|
272
|
+
self._set_warnings(warnings)
|
268
273
|
self.auth = auth or AuthConfig()
|
269
274
|
self.checks = checks or ChecksConfig()
|
270
275
|
self.phases = phases or PhasesConfig()
|
@@ -306,8 +311,17 @@ class OperationConfig(DiffBase):
|
|
306
311
|
request_cert=resolve(data.get("request-cert")),
|
307
312
|
request_cert_key=resolve(data.get("request-cert-key")),
|
308
313
|
parameters=load_parameters(data),
|
314
|
+
warnings=resolve_warnings(data.get("warnings")),
|
309
315
|
auth=AuthConfig.from_dict(data.get("auth", {})),
|
310
316
|
checks=ChecksConfig.from_dict(data.get("checks", {})),
|
311
317
|
phases=PhasesConfig.from_dict(data.get("phases", {})),
|
312
318
|
generation=GenerationConfig.from_dict(data.get("generation", {})),
|
313
319
|
)
|
320
|
+
|
321
|
+
def _set_warnings(self, warnings: bool | list[SchemathesisWarning] | None) -> None:
|
322
|
+
if warnings is False:
|
323
|
+
self.warnings = []
|
324
|
+
elif warnings is True:
|
325
|
+
self.warnings = list(SchemathesisWarning)
|
326
|
+
else:
|
327
|
+
self.warnings = warnings
|