schemathesis 3.35.5__py3-none-any.whl → 3.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/checks.py +8 -5
- schemathesis/cli/__init__.py +4 -13
- schemathesis/contrib/unique_data.py +1 -2
- schemathesis/generation/coverage.py +48 -7
- schemathesis/internal/checks.py +53 -0
- schemathesis/models.py +20 -8
- schemathesis/runner/__init__.py +7 -1
- schemathesis/runner/impl/context.py +18 -4
- schemathesis/runner/impl/core.py +57 -12
- schemathesis/runner/impl/solo.py +1 -1
- schemathesis/runner/impl/threadpool.py +3 -4
- schemathesis/schemas.py +2 -2
- schemathesis/specs/graphql/loaders.py +2 -2
- schemathesis/specs/graphql/schemas.py +9 -5
- schemathesis/specs/openapi/checks.py +76 -27
- schemathesis/specs/openapi/loaders.py +2 -2
- schemathesis/specs/openapi/schemas.py +19 -3
- schemathesis/stateful/config.py +1 -0
- schemathesis/stateful/context.py +10 -0
- schemathesis/stateful/runner.py +19 -2
- schemathesis/stateful/state_machine.py +2 -1
- schemathesis/stateful/validation.py +9 -4
- {schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/METADATA +1 -1
- {schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/RECORD +27 -26
- {schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/WHEEL +0 -0
- {schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/entry_points.txt +0 -0
- {schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/licenses/LICENSE +0 -0
schemathesis/checks.py
CHANGED
|
@@ -15,11 +15,12 @@ from .specs.openapi.checks import (
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
|
-
from .
|
|
18
|
+
from .internal.checks import CheckContext, CheckFunction
|
|
19
|
+
from .models import Case
|
|
19
20
|
from .transports.responses import GenericResponse
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
def not_a_server_error(response: GenericResponse, case: Case) -> bool | None:
|
|
23
|
+
def not_a_server_error(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
|
|
23
24
|
"""A check to verify that the response is not a server-side error."""
|
|
24
25
|
from .specs.graphql.schemas import GraphQLCase
|
|
25
26
|
from .specs.graphql.validation import validate_graphql_response
|
|
@@ -64,14 +65,16 @@ def register(check: CheckFunction) -> CheckFunction:
|
|
|
64
65
|
.. code-block:: python
|
|
65
66
|
|
|
66
67
|
@schemathesis.check
|
|
67
|
-
def new_check(response, case):
|
|
68
|
+
def new_check(ctx, response, case):
|
|
68
69
|
# some awesome assertions!
|
|
69
70
|
...
|
|
70
71
|
"""
|
|
71
72
|
from . import cli
|
|
73
|
+
from .internal.checks import wrap_check
|
|
72
74
|
|
|
75
|
+
_check = wrap_check(check)
|
|
73
76
|
global ALL_CHECKS
|
|
74
77
|
|
|
75
|
-
ALL_CHECKS += (
|
|
76
|
-
cli.CHECKS_TYPE.choices += (
|
|
78
|
+
ALL_CHECKS += (_check,)
|
|
79
|
+
cli.CHECKS_TYPE.choices += (_check.__name__,) # type: ignore
|
|
77
80
|
return check
|
schemathesis/cli/__init__.py
CHANGED
|
@@ -104,13 +104,6 @@ DEPRECATED_SHOW_ERROR_TRACEBACKS_OPTION_WARNING = (
|
|
|
104
104
|
"Warning: Option `--show-errors-tracebacks` is deprecated and will be removed in Schemathesis 4.0. "
|
|
105
105
|
"Use `--show-trace` instead"
|
|
106
106
|
)
|
|
107
|
-
DEPRECATED_CONTRIB_UNIQUE_DATA_OPTION_WARNING = (
|
|
108
|
-
"The `--contrib-unique-data` CLI option and the corresponding `schemathesis.contrib.unique_data` hook "
|
|
109
|
-
"are **DEPRECATED**. The concept of this feature does not fit the core principles of Hypothesis where "
|
|
110
|
-
"strategies are configurable on a per-example basis but this feature implies uniqueness across examples. "
|
|
111
|
-
"This leads to cryptic error messages about external state and flaky test runs, "
|
|
112
|
-
"therefore it will be removed in Schemathesis 4.0"
|
|
113
|
-
)
|
|
114
107
|
CASSETTES_PATH_INVALID_USAGE_MESSAGE = "Can't use `--store-network-log` and `--cassette-path` simultaneously"
|
|
115
108
|
COLOR_OPTIONS_INVALID_USAGE_MESSAGE = "Can't use `--no-color` and `--force-color` simultaneously"
|
|
116
109
|
PHASES_INVALID_USAGE_MESSAGE = "Can't use `--hypothesis-phases` and `--hypothesis-no-phases` simultaneously"
|
|
@@ -439,7 +432,7 @@ REPORT_TO_SERVICE = ReportToService()
|
|
|
439
432
|
"-A",
|
|
440
433
|
type=click.Choice(["basic", "digest"], case_sensitive=False),
|
|
441
434
|
default="basic",
|
|
442
|
-
help="Specify the authentication method",
|
|
435
|
+
help="Specify the authentication method. For custom authentication methods, see our Authentication documentation: https://schemathesis.readthedocs.io/en/stable/auth.html#custom-auth",
|
|
443
436
|
show_default=True,
|
|
444
437
|
metavar="",
|
|
445
438
|
)
|
|
@@ -949,9 +942,6 @@ def run(
|
|
|
949
942
|
entry for health_check in hypothesis_suppress_health_check for entry in health_check.as_hypothesis()
|
|
950
943
|
]
|
|
951
944
|
|
|
952
|
-
if contrib_unique_data:
|
|
953
|
-
click.secho(DEPRECATED_CONTRIB_UNIQUE_DATA_OPTION_WARNING, fg="yellow")
|
|
954
|
-
|
|
955
945
|
if show_errors_tracebacks:
|
|
956
946
|
click.secho(DEPRECATED_SHOW_ERROR_TRACEBACKS_OPTION_WARNING, fg="yellow")
|
|
957
947
|
show_trace = show_errors_tracebacks
|
|
@@ -1160,8 +1150,6 @@ def run(
|
|
|
1160
1150
|
else:
|
|
1161
1151
|
_fixups.install(fixups)
|
|
1162
1152
|
|
|
1163
|
-
if contrib_unique_data:
|
|
1164
|
-
contrib.unique_data.install()
|
|
1165
1153
|
if contrib_openapi_formats_uuid:
|
|
1166
1154
|
contrib.openapi.formats.uuid.install()
|
|
1167
1155
|
if contrib_openapi_fill_missing_examples:
|
|
@@ -1197,6 +1185,7 @@ def run(
|
|
|
1197
1185
|
seed=hypothesis_seed,
|
|
1198
1186
|
exit_first=exit_first,
|
|
1199
1187
|
max_failures=max_failures,
|
|
1188
|
+
unique_data=contrib_unique_data,
|
|
1200
1189
|
dry_run=dry_run,
|
|
1201
1190
|
store_interactions=cassette_path is not None,
|
|
1202
1191
|
checks=selected_checks,
|
|
@@ -1321,6 +1310,7 @@ def into_event_stream(
|
|
|
1321
1310
|
exit_first: bool,
|
|
1322
1311
|
max_failures: int | None,
|
|
1323
1312
|
rate_limit: str | None,
|
|
1313
|
+
unique_data: bool,
|
|
1324
1314
|
dry_run: bool,
|
|
1325
1315
|
store_interactions: bool,
|
|
1326
1316
|
stateful: Stateful | None,
|
|
@@ -1364,6 +1354,7 @@ def into_event_stream(
|
|
|
1364
1354
|
exit_first=exit_first,
|
|
1365
1355
|
max_failures=max_failures,
|
|
1366
1356
|
started_at=started_at,
|
|
1357
|
+
unique_data=unique_data,
|
|
1367
1358
|
dry_run=dry_run,
|
|
1368
1359
|
store_interactions=store_interactions,
|
|
1369
1360
|
checks=checks,
|
|
@@ -13,8 +13,7 @@ if TYPE_CHECKING:
|
|
|
13
13
|
|
|
14
14
|
def install() -> None:
|
|
15
15
|
warnings.warn(
|
|
16
|
-
"The
|
|
17
|
-
"are **DEPRECATED**. The concept of this feature does not fit the core principles of Hypothesis where "
|
|
16
|
+
"The `schemathesis.contrib.unique_data` hook is **DEPRECATED**. The concept of this feature does not fit the core principles of Hypothesis where "
|
|
18
17
|
"strategies are configurable on a per-example basis but this feature implies uniqueness across examples. "
|
|
19
18
|
"This leads to cryptic error messages about external state and flaky test runs, "
|
|
20
19
|
"therefore it will be removed in Schemathesis 4.0",
|
|
@@ -238,6 +238,8 @@ def _get_properties(schema: dict | bool) -> dict | bool:
|
|
|
238
238
|
if isinstance(schema, dict):
|
|
239
239
|
if "example" in schema:
|
|
240
240
|
return {"const": schema["example"]}
|
|
241
|
+
if "default" in schema:
|
|
242
|
+
return {"const": schema["default"]}
|
|
241
243
|
if schema.get("examples"):
|
|
242
244
|
return {"enum": schema["examples"]}
|
|
243
245
|
if schema.get("type") == "object":
|
|
@@ -265,12 +267,19 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
|
265
267
|
max_length = schema.get("maxLength")
|
|
266
268
|
example = schema.get("example")
|
|
267
269
|
examples = schema.get("examples")
|
|
268
|
-
|
|
270
|
+
default = schema.get("default")
|
|
271
|
+
if example or examples or default:
|
|
269
272
|
if example:
|
|
270
273
|
yield PositiveValue(example)
|
|
271
274
|
if examples:
|
|
272
275
|
for example in examples:
|
|
273
276
|
yield PositiveValue(example)
|
|
277
|
+
if (
|
|
278
|
+
default
|
|
279
|
+
and not (example is not None and default == example)
|
|
280
|
+
and not (examples is not None and any(default == ex for ex in examples))
|
|
281
|
+
):
|
|
282
|
+
yield PositiveValue(default)
|
|
274
283
|
elif not min_length and not max_length:
|
|
275
284
|
# Default positive value
|
|
276
285
|
yield PositiveValue(ctx.generate_from_schema(schema))
|
|
@@ -327,13 +336,20 @@ def _positive_number(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
|
327
336
|
multiple_of = schema.get("multipleOf")
|
|
328
337
|
example = schema.get("example")
|
|
329
338
|
examples = schema.get("examples")
|
|
339
|
+
default = schema.get("default")
|
|
330
340
|
|
|
331
|
-
if example or examples:
|
|
341
|
+
if example or examples or default:
|
|
332
342
|
if example:
|
|
333
343
|
yield PositiveValue(example)
|
|
334
344
|
if examples:
|
|
335
345
|
for example in examples:
|
|
336
346
|
yield PositiveValue(example)
|
|
347
|
+
if (
|
|
348
|
+
default
|
|
349
|
+
and not (example is not None and default == example)
|
|
350
|
+
and not (examples is not None and any(default == ex for ex in examples))
|
|
351
|
+
):
|
|
352
|
+
yield PositiveValue(default)
|
|
337
353
|
elif not minimum and not maximum:
|
|
338
354
|
# Default positive value
|
|
339
355
|
yield PositiveValue(ctx.generate_from_schema(schema))
|
|
@@ -382,13 +398,20 @@ def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Gener
|
|
|
382
398
|
seen = set()
|
|
383
399
|
example = schema.get("example")
|
|
384
400
|
examples = schema.get("examples")
|
|
401
|
+
default = schema.get("default")
|
|
385
402
|
|
|
386
|
-
if example or examples:
|
|
403
|
+
if example or examples or default:
|
|
387
404
|
if example:
|
|
388
405
|
yield PositiveValue(example)
|
|
389
406
|
if examples:
|
|
390
407
|
for example in examples:
|
|
391
408
|
yield PositiveValue(example)
|
|
409
|
+
if (
|
|
410
|
+
default
|
|
411
|
+
and not (example is not None and default == example)
|
|
412
|
+
and not (examples is not None and any(default == ex for ex in examples))
|
|
413
|
+
):
|
|
414
|
+
yield PositiveValue(default)
|
|
392
415
|
else:
|
|
393
416
|
yield PositiveValue(template)
|
|
394
417
|
seen.add(len(template))
|
|
@@ -425,19 +448,37 @@ def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Gener
|
|
|
425
448
|
def _positive_object(ctx: CoverageContext, schema: dict, template: dict) -> Generator[GeneratedValue, None, None]:
|
|
426
449
|
example = schema.get("example")
|
|
427
450
|
examples = schema.get("examples")
|
|
451
|
+
default = schema.get("default")
|
|
428
452
|
|
|
429
|
-
if example or examples:
|
|
453
|
+
if example or examples or default:
|
|
430
454
|
if example:
|
|
431
455
|
yield PositiveValue(example)
|
|
432
456
|
if examples:
|
|
433
457
|
for example in examples:
|
|
434
458
|
yield PositiveValue(example)
|
|
459
|
+
if (
|
|
460
|
+
default
|
|
461
|
+
and not (example is not None and default == example)
|
|
462
|
+
and not (examples is not None and any(default == ex for ex in examples))
|
|
463
|
+
):
|
|
464
|
+
yield PositiveValue(default)
|
|
465
|
+
|
|
435
466
|
else:
|
|
436
467
|
yield PositiveValue(template)
|
|
437
|
-
|
|
468
|
+
|
|
438
469
|
properties = schema.get("properties", {})
|
|
439
|
-
|
|
440
|
-
|
|
470
|
+
required = set(schema.get("required", []))
|
|
471
|
+
optional = list(set(properties) - required)
|
|
472
|
+
optional.sort()
|
|
473
|
+
|
|
474
|
+
# Generate combinations with required properties and one optional property
|
|
475
|
+
for name in optional:
|
|
476
|
+
combo = {k: v for k, v in template.items() if k in required or k == name}
|
|
477
|
+
if combo != template:
|
|
478
|
+
yield PositiveValue(combo)
|
|
479
|
+
# Generate only required properties
|
|
480
|
+
if set(properties) != required:
|
|
481
|
+
only_required = {k: v for k, v in template.items() if k in required}
|
|
441
482
|
yield PositiveValue(only_required)
|
|
442
483
|
seen = set()
|
|
443
484
|
for name, sub_schema in properties.items():
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import warnings
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING, Callable, Optional
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from ..models import Case
|
|
10
|
+
from ..transports.responses import GenericResponse
|
|
11
|
+
from requests.structures import CaseInsensitiveDict
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
CheckFunction = Callable[["CheckContext", "GenericResponse", "Case"], Optional[bool]]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class CheckContext:
|
|
19
|
+
"""Context for Schemathesis checks.
|
|
20
|
+
|
|
21
|
+
Provides access to broader test execution data beyond individual test cases.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
headers: CaseInsensitiveDict | None = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def wrap_check(check: Callable) -> CheckFunction:
|
|
28
|
+
"""Make older checks compatible with the new signature."""
|
|
29
|
+
signature = inspect.signature(check)
|
|
30
|
+
parameters = len(signature.parameters)
|
|
31
|
+
|
|
32
|
+
if parameters == 3:
|
|
33
|
+
# New style check, return as is
|
|
34
|
+
return check
|
|
35
|
+
|
|
36
|
+
if parameters == 2:
|
|
37
|
+
# Old style check, wrap it
|
|
38
|
+
warnings.warn(
|
|
39
|
+
f"The check function '{check.__name__}' uses an outdated signature. "
|
|
40
|
+
"Please update it to accept 'ctx' as the first argument: "
|
|
41
|
+
"(ctx: CheckContext, response: GenericResponse, case: Case) -> Optional[bool]",
|
|
42
|
+
DeprecationWarning,
|
|
43
|
+
stacklevel=2,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
def wrapper(_: CheckContext, response: GenericResponse, case: Case) -> Optional[bool]:
|
|
47
|
+
return check(response, case)
|
|
48
|
+
|
|
49
|
+
wrapper.__name__ = check.__name__
|
|
50
|
+
|
|
51
|
+
return wrapper
|
|
52
|
+
|
|
53
|
+
raise ValueError(f"Invalid check function signature. Expected 2 or 3 parameters, got {parameters}")
|
schemathesis/models.py
CHANGED
|
@@ -17,7 +17,6 @@ from typing import (
|
|
|
17
17
|
Generic,
|
|
18
18
|
Iterator,
|
|
19
19
|
NoReturn,
|
|
20
|
-
Optional,
|
|
21
20
|
Sequence,
|
|
22
21
|
Type,
|
|
23
22
|
TypeVar,
|
|
@@ -46,6 +45,7 @@ from .exceptions import (
|
|
|
46
45
|
)
|
|
47
46
|
from .generation import DataGenerationMethod, GenerationConfig, generate_random_case_id
|
|
48
47
|
from .hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher, dispatch
|
|
48
|
+
from .internal.checks import CheckContext
|
|
49
49
|
from .internal.copy import fast_deepcopy
|
|
50
50
|
from .internal.deprecation import deprecated_function, deprecated_property
|
|
51
51
|
from .internal.output import prepare_response_payload
|
|
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
|
|
|
65
65
|
|
|
66
66
|
from .auths import AuthStorage
|
|
67
67
|
from .failures import FailureContext
|
|
68
|
+
from .internal.checks import CheckFunction
|
|
68
69
|
from .schemas import BaseSchema
|
|
69
70
|
from .serializers import Serializer
|
|
70
71
|
from .stateful import Stateful, StatefulTest
|
|
@@ -421,6 +422,7 @@ class Case:
|
|
|
421
422
|
additional_checks: tuple[CheckFunction, ...] = (),
|
|
422
423
|
excluded_checks: tuple[CheckFunction, ...] = (),
|
|
423
424
|
code_sample_style: str | None = None,
|
|
425
|
+
headers: dict[str, Any] | None = None,
|
|
424
426
|
) -> None:
|
|
425
427
|
"""Validate application response.
|
|
426
428
|
|
|
@@ -434,17 +436,30 @@ class Case:
|
|
|
434
436
|
:param code_sample_style: Controls the style of code samples for failure reproduction.
|
|
435
437
|
"""
|
|
436
438
|
__tracebackhide__ = True
|
|
439
|
+
from requests.structures import CaseInsensitiveDict
|
|
440
|
+
|
|
437
441
|
from .checks import ALL_CHECKS
|
|
442
|
+
from .internal.checks import wrap_check
|
|
438
443
|
from .transports.responses import get_payload, get_reason
|
|
439
444
|
|
|
440
|
-
|
|
445
|
+
if checks:
|
|
446
|
+
_checks = tuple(wrap_check(check) for check in checks)
|
|
447
|
+
else:
|
|
448
|
+
_checks = checks
|
|
449
|
+
if additional_checks:
|
|
450
|
+
_additional_checks = tuple(wrap_check(check) for check in additional_checks)
|
|
451
|
+
else:
|
|
452
|
+
_additional_checks = additional_checks
|
|
453
|
+
|
|
454
|
+
checks = _checks or ALL_CHECKS
|
|
441
455
|
checks = tuple(check for check in checks if check not in excluded_checks)
|
|
442
|
-
additional_checks = tuple(check for check in
|
|
456
|
+
additional_checks = tuple(check for check in _additional_checks if check not in excluded_checks)
|
|
443
457
|
failed_checks = []
|
|
458
|
+
ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
|
|
444
459
|
for check in chain(checks, additional_checks):
|
|
445
460
|
copied_case = self.partial_deepcopy()
|
|
446
461
|
try:
|
|
447
|
-
check(response, copied_case)
|
|
462
|
+
check(ctx, response, copied_case)
|
|
448
463
|
except AssertionError as exc:
|
|
449
464
|
maybe_set_assertion_message(exc, check.__name__)
|
|
450
465
|
failed_checks.append(exc)
|
|
@@ -514,7 +529,7 @@ class Case:
|
|
|
514
529
|
) -> requests.Response:
|
|
515
530
|
__tracebackhide__ = True
|
|
516
531
|
response = self.call(base_url, session, headers, **kwargs)
|
|
517
|
-
self.validate_response(response, checks, code_sample_style=code_sample_style)
|
|
532
|
+
self.validate_response(response, checks, code_sample_style=code_sample_style, headers=headers)
|
|
518
533
|
return response
|
|
519
534
|
|
|
520
535
|
def _get_url(self, base_url: str | None) -> str:
|
|
@@ -1233,6 +1248,3 @@ class TestResultSet:
|
|
|
1233
1248
|
def add_warning(self, warning: str) -> None:
|
|
1234
1249
|
"""Add a new warning to the warnings list."""
|
|
1235
1250
|
self.warnings.append(warning)
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
CheckFunction = Callable[["GenericResponse", Case], Optional[bool]]
|
schemathesis/runner/__init__.py
CHANGED
|
@@ -347,6 +347,7 @@ def from_schema(
|
|
|
347
347
|
exit_first: bool = False,
|
|
348
348
|
max_failures: int | None = None,
|
|
349
349
|
started_at: str | None = None,
|
|
350
|
+
unique_data: bool = False,
|
|
350
351
|
dry_run: bool = False,
|
|
351
352
|
store_interactions: bool = False,
|
|
352
353
|
stateful: Stateful | None = None,
|
|
@@ -373,7 +374,6 @@ def from_schema(
|
|
|
373
374
|
probe_config = probe_config or ProbeConfig()
|
|
374
375
|
|
|
375
376
|
hypothesis_settings = hypothesis_settings or hypothesis.settings(deadline=DEFAULT_DEADLINE)
|
|
376
|
-
generation_config = generation_config or GenerationConfig()
|
|
377
377
|
request_config = RequestConfig(
|
|
378
378
|
timeout=request_timeout,
|
|
379
379
|
tls_verify=request_tls_verify,
|
|
@@ -405,6 +405,7 @@ def from_schema(
|
|
|
405
405
|
exit_first=exit_first,
|
|
406
406
|
max_failures=max_failures,
|
|
407
407
|
started_at=started_at,
|
|
408
|
+
unique_data=unique_data,
|
|
408
409
|
dry_run=dry_run,
|
|
409
410
|
store_interactions=store_interactions,
|
|
410
411
|
stateful=stateful,
|
|
@@ -430,6 +431,7 @@ def from_schema(
|
|
|
430
431
|
exit_first=exit_first,
|
|
431
432
|
max_failures=max_failures,
|
|
432
433
|
started_at=started_at,
|
|
434
|
+
unique_data=unique_data,
|
|
433
435
|
dry_run=dry_run,
|
|
434
436
|
store_interactions=store_interactions,
|
|
435
437
|
stateful=stateful,
|
|
@@ -455,6 +457,7 @@ def from_schema(
|
|
|
455
457
|
exit_first=exit_first,
|
|
456
458
|
max_failures=max_failures,
|
|
457
459
|
started_at=started_at,
|
|
460
|
+
unique_data=unique_data,
|
|
458
461
|
dry_run=dry_run,
|
|
459
462
|
store_interactions=store_interactions,
|
|
460
463
|
stateful=stateful,
|
|
@@ -481,6 +484,7 @@ def from_schema(
|
|
|
481
484
|
exit_first=exit_first,
|
|
482
485
|
max_failures=max_failures,
|
|
483
486
|
started_at=started_at,
|
|
487
|
+
unique_data=unique_data,
|
|
484
488
|
dry_run=dry_run,
|
|
485
489
|
store_interactions=store_interactions,
|
|
486
490
|
stateful=stateful,
|
|
@@ -506,6 +510,7 @@ def from_schema(
|
|
|
506
510
|
exit_first=exit_first,
|
|
507
511
|
max_failures=max_failures,
|
|
508
512
|
started_at=started_at,
|
|
513
|
+
unique_data=unique_data,
|
|
509
514
|
dry_run=dry_run,
|
|
510
515
|
store_interactions=store_interactions,
|
|
511
516
|
stateful=stateful,
|
|
@@ -530,6 +535,7 @@ def from_schema(
|
|
|
530
535
|
exit_first=exit_first,
|
|
531
536
|
max_failures=max_failures,
|
|
532
537
|
started_at=started_at,
|
|
538
|
+
unique_data=unique_data,
|
|
533
539
|
dry_run=dry_run,
|
|
534
540
|
store_interactions=store_interactions,
|
|
535
541
|
stateful=stateful,
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
4
5
|
|
|
6
|
+
from ...constants import NOT_SET
|
|
5
7
|
from ...models import TestResult, TestResultSet
|
|
6
|
-
from typing import TYPE_CHECKING
|
|
7
8
|
|
|
8
9
|
if TYPE_CHECKING:
|
|
9
|
-
from ...exceptions import OperationSchemaError
|
|
10
10
|
import threading
|
|
11
11
|
|
|
12
|
+
from ...exceptions import OperationSchemaError
|
|
13
|
+
from ...models import Case
|
|
14
|
+
from ...types import NotSet
|
|
15
|
+
|
|
12
16
|
|
|
13
17
|
@dataclass
|
|
14
18
|
class RunnerContext:
|
|
@@ -17,13 +21,17 @@ class RunnerContext:
|
|
|
17
21
|
data: TestResultSet
|
|
18
22
|
seed: int | None
|
|
19
23
|
stop_event: threading.Event
|
|
24
|
+
unique_data: bool
|
|
25
|
+
outcome_cache: dict[int, BaseException | None]
|
|
20
26
|
|
|
21
|
-
__slots__ = ("data", "seed", "stop_event")
|
|
27
|
+
__slots__ = ("data", "seed", "stop_event", "unique_data", "outcome_cache")
|
|
22
28
|
|
|
23
|
-
def __init__(self, seed: int | None, stop_event: threading.Event) -> None:
|
|
29
|
+
def __init__(self, *, seed: int | None, stop_event: threading.Event, unique_data: bool) -> None:
|
|
24
30
|
self.data = TestResultSet(seed=seed)
|
|
25
31
|
self.seed = seed
|
|
26
32
|
self.stop_event = stop_event
|
|
33
|
+
self.outcome_cache = {}
|
|
34
|
+
self.unique_data = unique_data
|
|
27
35
|
|
|
28
36
|
@property
|
|
29
37
|
def is_stopped(self) -> bool:
|
|
@@ -54,5 +62,11 @@ class RunnerContext:
|
|
|
54
62
|
def add_warning(self, message: str) -> None:
|
|
55
63
|
self.data.add_warning(message)
|
|
56
64
|
|
|
65
|
+
def cache_outcome(self, case: Case, outcome: BaseException | None) -> None:
|
|
66
|
+
self.outcome_cache[hash(case)] = outcome
|
|
67
|
+
|
|
68
|
+
def get_cached_outcome(self, case: Case) -> BaseException | None | NotSet:
|
|
69
|
+
return self.outcome_cache.get(hash(case), NOT_SET)
|
|
70
|
+
|
|
57
71
|
|
|
58
72
|
ALL_NOT_FOUND_WARNING_MESSAGE = "All API responses have a 404 status code. Did you specify the proper API location?"
|
schemathesis/runner/impl/core.py
CHANGED
|
@@ -21,6 +21,7 @@ from hypothesis.errors import HypothesisException, InvalidArgument
|
|
|
21
21
|
from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
|
|
22
22
|
from jsonschema.exceptions import SchemaError as JsonSchemaError
|
|
23
23
|
from jsonschema.exceptions import ValidationError
|
|
24
|
+
from requests.structures import CaseInsensitiveDict
|
|
24
25
|
from urllib3.exceptions import InsecureRequestWarning
|
|
25
26
|
|
|
26
27
|
from ... import experimental, failures, hooks
|
|
@@ -56,9 +57,10 @@ from ...exceptions import (
|
|
|
56
57
|
)
|
|
57
58
|
from ...generation import DataGenerationMethod, GenerationConfig
|
|
58
59
|
from ...hooks import HookContext, get_all_by_name
|
|
60
|
+
from ...internal.checks import CheckContext
|
|
59
61
|
from ...internal.datetime import current_datetime
|
|
60
62
|
from ...internal.result import Err, Ok, Result
|
|
61
|
-
from ...models import APIOperation, Case, Check,
|
|
63
|
+
from ...models import APIOperation, Case, Check, Status, TestResult
|
|
62
64
|
from ...runner import events
|
|
63
65
|
from ...service import extensions
|
|
64
66
|
from ...service.models import AnalysisResult, AnalysisSuccess
|
|
@@ -75,13 +77,16 @@ from ..serialization import SerializedTestResult
|
|
|
75
77
|
from .context import RunnerContext
|
|
76
78
|
|
|
77
79
|
if TYPE_CHECKING:
|
|
78
|
-
from ...types import RawAuth
|
|
79
|
-
from ...schemas import BaseSchema
|
|
80
|
-
from ..._override import CaseOverride
|
|
81
|
-
from requests.auth import HTTPDigestAuth
|
|
82
80
|
from types import TracebackType
|
|
81
|
+
|
|
82
|
+
from requests.auth import HTTPDigestAuth
|
|
83
|
+
|
|
84
|
+
from ..._override import CaseOverride
|
|
85
|
+
from ...internal.checks import CheckFunction
|
|
86
|
+
from ...schemas import BaseSchema
|
|
83
87
|
from ...service.client import ServiceClient
|
|
84
88
|
from ...transports.responses import GenericResponse, WSGIResponse
|
|
89
|
+
from ...types import RawAuth
|
|
85
90
|
|
|
86
91
|
|
|
87
92
|
def _should_count_towards_stop(event: events.ExecutionEvent) -> bool:
|
|
@@ -95,7 +100,7 @@ class BaseRunner:
|
|
|
95
100
|
max_response_time: int | None
|
|
96
101
|
targets: Iterable[Target]
|
|
97
102
|
hypothesis_settings: hypothesis.settings
|
|
98
|
-
generation_config: GenerationConfig
|
|
103
|
+
generation_config: GenerationConfig | None
|
|
99
104
|
probe_config: probes.ProbeConfig
|
|
100
105
|
request_config: RequestConfig = field(default_factory=RequestConfig)
|
|
101
106
|
override: CaseOverride | None = None
|
|
@@ -107,6 +112,7 @@ class BaseRunner:
|
|
|
107
112
|
exit_first: bool = False
|
|
108
113
|
max_failures: int | None = None
|
|
109
114
|
started_at: str = field(default_factory=current_datetime)
|
|
115
|
+
unique_data: bool = False
|
|
110
116
|
dry_run: bool = False
|
|
111
117
|
stateful: Stateful | None = None
|
|
112
118
|
stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT
|
|
@@ -125,7 +131,7 @@ class BaseRunner:
|
|
|
125
131
|
# If auth is explicitly provided, then the global provider is ignored
|
|
126
132
|
if self.auth is not None:
|
|
127
133
|
unregister_auth()
|
|
128
|
-
ctx = RunnerContext(self.seed, stop_event)
|
|
134
|
+
ctx = RunnerContext(seed=self.seed, stop_event=stop_event, unique_data=self.unique_data)
|
|
129
135
|
start_time = time.monotonic()
|
|
130
136
|
initialized = None
|
|
131
137
|
__probes = None
|
|
@@ -333,7 +339,7 @@ class BaseRunner:
|
|
|
333
339
|
maker: Callable,
|
|
334
340
|
test_func: Callable,
|
|
335
341
|
settings: hypothesis.settings,
|
|
336
|
-
generation_config: GenerationConfig,
|
|
342
|
+
generation_config: GenerationConfig | None,
|
|
337
343
|
ctx: RunnerContext,
|
|
338
344
|
recursion_level: int = 0,
|
|
339
345
|
headers: dict[str, Any] | None = None,
|
|
@@ -561,9 +567,10 @@ def run_test(
|
|
|
561
567
|
try:
|
|
562
568
|
with catch_warnings(record=True) as warnings, capture_hypothesis_output() as hypothesis_output:
|
|
563
569
|
test(
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
570
|
+
ctx=ctx,
|
|
571
|
+
checks=checks,
|
|
572
|
+
targets=targets,
|
|
573
|
+
result=result,
|
|
567
574
|
errors=errors,
|
|
568
575
|
headers=headers,
|
|
569
576
|
data_generation_methods=data_generation_methods,
|
|
@@ -789,6 +796,7 @@ def deduplicate_errors(errors: list[Exception]) -> Generator[Exception, None, No
|
|
|
789
796
|
def run_checks(
|
|
790
797
|
*,
|
|
791
798
|
case: Case,
|
|
799
|
+
ctx: CheckContext,
|
|
792
800
|
checks: Iterable[CheckFunction],
|
|
793
801
|
check_results: list[Check],
|
|
794
802
|
result: TestResult,
|
|
@@ -811,7 +819,7 @@ def run_checks(
|
|
|
811
819
|
check_name = check.__name__
|
|
812
820
|
copied_case = case.partial_deepcopy()
|
|
813
821
|
try:
|
|
814
|
-
skip_check = check(response, copied_case)
|
|
822
|
+
skip_check = check(ctx, response, copied_case)
|
|
815
823
|
if not skip_check:
|
|
816
824
|
check_result = result.add_success(check_name, copied_case, response, elapsed_time)
|
|
817
825
|
check_results.append(check_result)
|
|
@@ -897,7 +905,33 @@ def _force_data_generation_method(values: list[DataGenerationMethod], case: Case
|
|
|
897
905
|
values[:] = [data_generation_method]
|
|
898
906
|
|
|
899
907
|
|
|
908
|
+
def cached_test_func(f: Callable) -> Callable:
|
|
909
|
+
def wrapped(*, ctx: RunnerContext, case: Case, **kwargs: Any) -> None:
|
|
910
|
+
if ctx.unique_data:
|
|
911
|
+
cached = ctx.get_cached_outcome(case)
|
|
912
|
+
if isinstance(cached, BaseException):
|
|
913
|
+
raise cached
|
|
914
|
+
elif cached is None:
|
|
915
|
+
return None
|
|
916
|
+
try:
|
|
917
|
+
f(ctx=ctx, case=case, **kwargs)
|
|
918
|
+
except BaseException as exc:
|
|
919
|
+
ctx.cache_outcome(case, exc)
|
|
920
|
+
raise
|
|
921
|
+
else:
|
|
922
|
+
ctx.cache_outcome(case, None)
|
|
923
|
+
else:
|
|
924
|
+
f(ctx=ctx, case=case, **kwargs)
|
|
925
|
+
|
|
926
|
+
wrapped.__name__ = f.__name__
|
|
927
|
+
|
|
928
|
+
return wrapped
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
@cached_test_func
|
|
900
932
|
def network_test(
|
|
933
|
+
*,
|
|
934
|
+
ctx: RunnerContext,
|
|
901
935
|
case: Case,
|
|
902
936
|
checks: Iterable[CheckFunction],
|
|
903
937
|
targets: Iterable[Target],
|
|
@@ -980,9 +1014,12 @@ def _network_test(
|
|
|
980
1014
|
context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
|
|
981
1015
|
run_targets(targets, context)
|
|
982
1016
|
status = Status.success
|
|
1017
|
+
|
|
1018
|
+
ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
|
|
983
1019
|
try:
|
|
984
1020
|
run_checks(
|
|
985
1021
|
case=case,
|
|
1022
|
+
ctx=ctx,
|
|
986
1023
|
checks=checks,
|
|
987
1024
|
check_results=check_results,
|
|
988
1025
|
result=result,
|
|
@@ -1009,7 +1046,9 @@ def get_session(auth: HTTPDigestAuth | RawAuth | None = None) -> Generator[reque
|
|
|
1009
1046
|
yield session
|
|
1010
1047
|
|
|
1011
1048
|
|
|
1049
|
+
@cached_test_func
|
|
1012
1050
|
def wsgi_test(
|
|
1051
|
+
ctx: RunnerContext,
|
|
1013
1052
|
case: Case,
|
|
1014
1053
|
checks: Iterable[CheckFunction],
|
|
1015
1054
|
targets: Iterable[Target],
|
|
@@ -1066,9 +1105,11 @@ def _wsgi_test(
|
|
|
1066
1105
|
result.logs.extend(recorded.records)
|
|
1067
1106
|
status = Status.success
|
|
1068
1107
|
check_results: list[Check] = []
|
|
1108
|
+
ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
|
|
1069
1109
|
try:
|
|
1070
1110
|
run_checks(
|
|
1071
1111
|
case=case,
|
|
1112
|
+
ctx=ctx,
|
|
1072
1113
|
checks=checks,
|
|
1073
1114
|
check_results=check_results,
|
|
1074
1115
|
result=result,
|
|
@@ -1087,7 +1128,9 @@ def _wsgi_test(
|
|
|
1087
1128
|
return response
|
|
1088
1129
|
|
|
1089
1130
|
|
|
1131
|
+
@cached_test_func
|
|
1090
1132
|
def asgi_test(
|
|
1133
|
+
ctx: RunnerContext,
|
|
1091
1134
|
case: Case,
|
|
1092
1135
|
checks: Iterable[CheckFunction],
|
|
1093
1136
|
targets: Iterable[Target],
|
|
@@ -1140,9 +1183,11 @@ def _asgi_test(
|
|
|
1140
1183
|
run_targets(targets, context)
|
|
1141
1184
|
status = Status.success
|
|
1142
1185
|
check_results: list[Check] = []
|
|
1186
|
+
ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
|
|
1143
1187
|
try:
|
|
1144
1188
|
run_checks(
|
|
1145
1189
|
case=case,
|
|
1190
|
+
ctx=ctx,
|
|
1146
1191
|
checks=checks,
|
|
1147
1192
|
check_results=check_results,
|
|
1148
1193
|
result=result,
|
schemathesis/runner/impl/solo.py
CHANGED