schemathesis 4.0.0a9__py3-none-any.whl → 4.0.0a11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/__init__.py +3 -7
- schemathesis/checks.py +17 -7
- schemathesis/cli/commands/__init__.py +51 -3
- schemathesis/cli/commands/data.py +10 -0
- schemathesis/cli/commands/run/__init__.py +147 -260
- schemathesis/cli/commands/run/context.py +2 -3
- schemathesis/cli/commands/run/events.py +4 -0
- schemathesis/cli/commands/run/executor.py +60 -73
- schemathesis/cli/commands/run/filters.py +15 -165
- schemathesis/cli/commands/run/handlers/cassettes.py +105 -104
- schemathesis/cli/commands/run/handlers/junitxml.py +6 -5
- schemathesis/cli/commands/run/handlers/output.py +26 -47
- schemathesis/cli/commands/run/loaders.py +35 -50
- schemathesis/cli/commands/run/validation.py +36 -161
- schemathesis/cli/core.py +5 -3
- schemathesis/cli/ext/fs.py +7 -5
- schemathesis/cli/ext/options.py +0 -21
- schemathesis/config/__init__.py +188 -0
- schemathesis/config/_auth.py +51 -0
- schemathesis/config/_checks.py +268 -0
- schemathesis/config/_diff_base.py +99 -0
- schemathesis/config/_env.py +21 -0
- schemathesis/config/_error.py +156 -0
- schemathesis/config/_generation.py +150 -0
- schemathesis/config/_health_check.py +24 -0
- schemathesis/config/_operations.py +313 -0
- schemathesis/config/_output.py +171 -0
- schemathesis/config/_parameters.py +19 -0
- schemathesis/config/_phases.py +151 -0
- schemathesis/config/_projects.py +495 -0
- schemathesis/config/_rate_limit.py +17 -0
- schemathesis/config/_report.py +116 -0
- schemathesis/config/_validator.py +9 -0
- schemathesis/config/schema.json +837 -0
- schemathesis/core/__init__.py +3 -0
- schemathesis/core/compat.py +16 -9
- schemathesis/core/errors.py +19 -2
- schemathesis/core/failures.py +6 -7
- schemathesis/core/hooks.py +20 -0
- schemathesis/core/output/__init__.py +14 -37
- schemathesis/core/output/sanitization.py +3 -146
- schemathesis/core/validation.py +16 -0
- schemathesis/engine/__init__.py +2 -4
- schemathesis/engine/context.py +41 -43
- schemathesis/engine/core.py +7 -5
- schemathesis/engine/phases/__init__.py +10 -0
- schemathesis/engine/phases/probes.py +8 -8
- schemathesis/engine/phases/stateful/_executor.py +68 -43
- schemathesis/engine/phases/unit/__init__.py +23 -15
- schemathesis/engine/phases/unit/_executor.py +77 -17
- schemathesis/engine/phases/unit/_pool.py +1 -1
- schemathesis/errors.py +2 -0
- schemathesis/filters.py +2 -3
- schemathesis/generation/__init__.py +6 -31
- schemathesis/generation/case.py +5 -3
- schemathesis/generation/coverage.py +174 -134
- schemathesis/generation/hypothesis/__init__.py +7 -1
- schemathesis/generation/hypothesis/builder.py +40 -14
- schemathesis/generation/meta.py +3 -3
- schemathesis/generation/overrides.py +37 -1
- schemathesis/generation/stateful/state_machine.py +8 -1
- schemathesis/graphql/loaders.py +21 -12
- schemathesis/openapi/checks.py +12 -8
- schemathesis/openapi/generation/filters.py +10 -8
- schemathesis/openapi/loaders.py +22 -13
- schemathesis/pytest/lazy.py +2 -5
- schemathesis/pytest/plugin.py +11 -2
- schemathesis/schemas.py +13 -61
- schemathesis/specs/graphql/schemas.py +11 -15
- schemathesis/specs/openapi/_hypothesis.py +12 -8
- schemathesis/specs/openapi/checks.py +16 -18
- schemathesis/specs/openapi/examples.py +4 -3
- schemathesis/specs/openapi/formats.py +2 -2
- schemathesis/specs/openapi/negative/__init__.py +2 -2
- schemathesis/specs/openapi/patterns.py +46 -16
- schemathesis/specs/openapi/references.py +2 -3
- schemathesis/specs/openapi/schemas.py +11 -20
- schemathesis/specs/openapi/stateful/__init__.py +10 -5
- schemathesis/transport/prepare.py +7 -6
- schemathesis/transport/requests.py +3 -1
- schemathesis/transport/wsgi.py +3 -4
- {schemathesis-4.0.0a9.dist-info → schemathesis-4.0.0a11.dist-info}/METADATA +7 -8
- schemathesis-4.0.0a11.dist-info/RECORD +166 -0
- schemathesis/cli/commands/run/checks.py +0 -79
- schemathesis/cli/commands/run/hypothesis.py +0 -78
- schemathesis/cli/commands/run/reports.py +0 -72
- schemathesis/cli/hooks.py +0 -36
- schemathesis/engine/config.py +0 -59
- schemathesis/experimental/__init__.py +0 -72
- schemathesis-4.0.0a9.dist-info/RECORD +0 -153
- {schemathesis-4.0.0a9.dist-info → schemathesis-4.0.0a11.dist-info}/WHEEL +0 -0
- {schemathesis-4.0.0a9.dist-info → schemathesis-4.0.0a11.dist-info}/entry_points.txt +0 -0
- {schemathesis-4.0.0a9.dist-info → schemathesis-4.0.0a11.dist-info}/licenses/LICENSE +0 -0
@@ -20,7 +20,6 @@ from schemathesis.engine import Status, events
|
|
20
20
|
if TYPE_CHECKING:
|
21
21
|
import requests
|
22
22
|
|
23
|
-
from schemathesis.engine.config import NetworkConfig
|
24
23
|
from schemathesis.engine.context import EngineContext
|
25
24
|
from schemathesis.engine.events import EventGenerator
|
26
25
|
from schemathesis.engine.phases import Phase
|
@@ -36,7 +35,7 @@ class ProbePayload:
|
|
36
35
|
|
37
36
|
def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
|
38
37
|
"""Discover capabilities of the tested app."""
|
39
|
-
probes = run(ctx
|
38
|
+
probes = run(ctx)
|
40
39
|
status = Status.SUCCESS
|
41
40
|
payload: Result[ProbePayload, Exception] | None = None
|
42
41
|
for result in probes:
|
@@ -44,7 +43,7 @@ def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
|
|
44
43
|
from ...specs.openapi import formats
|
45
44
|
from ...specs.openapi.formats import HEADER_FORMAT, header_values
|
46
45
|
|
47
|
-
formats.register(HEADER_FORMAT, header_values(
|
46
|
+
formats.register(HEADER_FORMAT, header_values(exclude_characters="\n\r\x00"))
|
48
47
|
if result.error is not None:
|
49
48
|
status = Status.ERROR
|
50
49
|
payload = Err(result.error)
|
@@ -54,9 +53,9 @@ def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
|
|
54
53
|
yield events.PhaseFinished(phase=phase, status=status, payload=payload)
|
55
54
|
|
56
55
|
|
57
|
-
def run(
|
56
|
+
def run(ctx: EngineContext) -> list[ProbeRun]:
|
58
57
|
"""Run all probes against the given schema."""
|
59
|
-
return [send(probe(),
|
58
|
+
return [send(probe(), ctx) for probe in PROBES]
|
60
59
|
|
61
60
|
|
62
61
|
HEADER_NAME = "X-Schemathesis-Probe"
|
@@ -124,19 +123,20 @@ class NullByteInHeader(Probe):
|
|
124
123
|
PROBES = (NullByteInHeader,)
|
125
124
|
|
126
125
|
|
127
|
-
def send(probe: Probe,
|
126
|
+
def send(probe: Probe, ctx: EngineContext) -> ProbeRun:
|
128
127
|
"""Send the probe to the application."""
|
129
128
|
from requests import PreparedRequest, Request, RequestException
|
130
129
|
from requests.exceptions import MissingSchema
|
131
130
|
from urllib3.exceptions import InsecureRequestWarning
|
132
131
|
|
133
132
|
try:
|
134
|
-
|
133
|
+
session = ctx.get_session()
|
134
|
+
request = probe.prepare_request(session, Request(), ctx.schema)
|
135
135
|
request.headers[HEADER_NAME] = probe.name
|
136
136
|
request.headers["User-Agent"] = USER_AGENT
|
137
137
|
with warnings.catch_warnings():
|
138
138
|
warnings.simplefilter("ignore", InsecureRequestWarning)
|
139
|
-
response = session.send(request, timeout=config.
|
139
|
+
response = session.send(request, timeout=ctx.config.request_timeout or 2)
|
140
140
|
except MissingSchema:
|
141
141
|
# In-process ASGI/WSGI testing will have local URLs and requires extra handling
|
142
142
|
# which is not currently implemented
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
3
3
|
import queue
|
4
4
|
import time
|
5
5
|
import unittest
|
6
|
-
from dataclasses import
|
6
|
+
from dataclasses import dataclass
|
7
7
|
from typing import Any
|
8
8
|
from warnings import catch_warnings
|
9
9
|
|
@@ -11,6 +11,7 @@ import hypothesis
|
|
11
11
|
from hypothesis.control import current_build_context
|
12
12
|
from hypothesis.errors import Flaky, Unsatisfiable
|
13
13
|
from hypothesis.stateful import Rule
|
14
|
+
from requests.structures import CaseInsensitiveDict
|
14
15
|
|
15
16
|
from schemathesis.checks import CheckContext, CheckFunction, run_checks
|
16
17
|
from schemathesis.core.failures import Failure, FailureGroup
|
@@ -21,6 +22,7 @@ from schemathesis.engine.control import ExecutionControl
|
|
21
22
|
from schemathesis.engine.phases import PhaseName
|
22
23
|
from schemathesis.engine.phases.stateful.context import StatefulContext
|
23
24
|
from schemathesis.engine.recorder import ScenarioRecorder
|
25
|
+
from schemathesis.generation import overrides
|
24
26
|
from schemathesis.generation.case import Case
|
25
27
|
from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
|
26
28
|
from schemathesis.generation.stateful.state_machine import (
|
@@ -47,6 +49,17 @@ def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> d
|
|
47
49
|
return kwargs
|
48
50
|
|
49
51
|
|
52
|
+
@dataclass
|
53
|
+
class CachedCheckContextData:
|
54
|
+
override: Any
|
55
|
+
auth: Any
|
56
|
+
headers: Any
|
57
|
+
config: Any
|
58
|
+
transport_kwargs: Any
|
59
|
+
|
60
|
+
__slots__ = ("override", "auth", "headers", "config", "transport_kwargs")
|
61
|
+
|
62
|
+
|
50
63
|
def execute_state_machine_loop(
|
51
64
|
*,
|
52
65
|
state_machine: type[APIStateMachine],
|
@@ -54,21 +67,15 @@ def execute_state_machine_loop(
|
|
54
67
|
engine: EngineContext,
|
55
68
|
) -> None:
|
56
69
|
"""Execute the state machine testing loop."""
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
execution=replace(
|
62
|
-
engine.config.execution,
|
63
|
-
hypothesis_settings=hypothesis.settings(engine.config.execution.hypothesis_settings, **kwargs),
|
64
|
-
),
|
65
|
-
)
|
66
|
-
else:
|
67
|
-
config = engine.config
|
70
|
+
configured_hypothesis_settings = engine.config.get_hypothesis_settings(phase="stateful")
|
71
|
+
kwargs = _get_hypothesis_settings_kwargs_override(configured_hypothesis_settings)
|
72
|
+
hypothesis_settings = hypothesis.settings(configured_hypothesis_settings, **kwargs)
|
73
|
+
generation = engine.config.generation_for(phase="stateful")
|
68
74
|
|
69
|
-
ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=
|
75
|
+
ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=generation.maximize))
|
70
76
|
|
71
|
-
|
77
|
+
# Caches for validate_response to avoid repeated config lookups per operation
|
78
|
+
_check_context_cache: dict[str, CachedCheckContextData] = {}
|
72
79
|
|
73
80
|
class _InstrumentedStateMachine(state_machine): # type: ignore[valid-type,misc]
|
74
81
|
"""State machine with additional hooks for emitting events."""
|
@@ -78,23 +85,22 @@ def execute_state_machine_loop(
|
|
78
85
|
self._start_time = time.monotonic()
|
79
86
|
self._scenario_id = scenario_started.id
|
80
87
|
event_queue.put(scenario_started)
|
81
|
-
self._check_ctx = engine.get_check_context(self.recorder)
|
82
88
|
|
83
89
|
def get_call_kwargs(self, case: Case) -> dict[str, Any]:
|
84
|
-
return
|
90
|
+
return engine.get_transport_kwargs(operation=case.operation)
|
85
91
|
|
86
92
|
def _repr_step(self, rule: Rule, data: dict, result: StepOutput) -> str:
|
87
93
|
return ""
|
88
94
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
def before_call(self, case: Case) -> None:
|
96
|
+
override = overrides.for_operation(engine.config, operation=case.operation)
|
97
|
+
for location in ("query", "headers", "cookies", "path_parameters"):
|
98
|
+
entry = getattr(override, location)
|
99
|
+
if entry:
|
100
|
+
container = getattr(case, location) or {}
|
101
|
+
container.update(entry)
|
102
|
+
setattr(case, location, container)
|
103
|
+
return super().before_call(case)
|
98
104
|
|
99
105
|
def step(self, input: StepInput) -> StepOutput | None:
|
100
106
|
# Checking the stop event once inside `step` is sufficient as it is called frequently
|
@@ -102,7 +108,7 @@ def execute_state_machine_loop(
|
|
102
108
|
if engine.has_to_stop:
|
103
109
|
raise KeyboardInterrupt
|
104
110
|
try:
|
105
|
-
if
|
111
|
+
if generation.unique_inputs:
|
106
112
|
cached = ctx.get_step_outcome(input.case)
|
107
113
|
if isinstance(cached, BaseException):
|
108
114
|
raise cached
|
@@ -111,13 +117,13 @@ def execute_state_machine_loop(
|
|
111
117
|
result = super().step(input)
|
112
118
|
ctx.step_succeeded()
|
113
119
|
except FailureGroup as exc:
|
114
|
-
if
|
120
|
+
if generation.unique_inputs:
|
115
121
|
for failure in exc.exceptions:
|
116
122
|
ctx.store_step_outcome(input.case, failure)
|
117
123
|
ctx.step_failed()
|
118
124
|
raise
|
119
125
|
except Exception as exc:
|
120
|
-
if
|
126
|
+
if generation.unique_inputs:
|
121
127
|
ctx.store_step_outcome(input.case, exc)
|
122
128
|
ctx.step_errored()
|
123
129
|
raise
|
@@ -125,11 +131,11 @@ def execute_state_machine_loop(
|
|
125
131
|
ctx.step_interrupted()
|
126
132
|
raise
|
127
133
|
except BaseException as exc:
|
128
|
-
if
|
134
|
+
if generation.unique_inputs:
|
129
135
|
ctx.store_step_outcome(input.case, exc)
|
130
136
|
raise exc
|
131
137
|
else:
|
132
|
-
if
|
138
|
+
if generation.unique_inputs:
|
133
139
|
ctx.store_step_outcome(input.case, None)
|
134
140
|
return result
|
135
141
|
|
@@ -139,12 +145,34 @@ def execute_state_machine_loop(
|
|
139
145
|
self.recorder.record_response(case_id=case.id, response=response)
|
140
146
|
ctx.collect_metric(case, response)
|
141
147
|
ctx.current_response = response
|
148
|
+
|
149
|
+
label = case.operation.label
|
150
|
+
cached = _check_context_cache.get(label)
|
151
|
+
if cached is None:
|
152
|
+
headers = engine.config.headers_for(operation=case.operation)
|
153
|
+
cached = CachedCheckContextData(
|
154
|
+
override=overrides.for_operation(engine.config, operation=case.operation),
|
155
|
+
auth=engine.config.auth_for(operation=case.operation),
|
156
|
+
headers=CaseInsensitiveDict(headers) if headers else None,
|
157
|
+
config=engine.config.checks_config_for(operation=case.operation, phase="stateful"),
|
158
|
+
transport_kwargs=engine.get_transport_kwargs(operation=case.operation),
|
159
|
+
)
|
160
|
+
_check_context_cache[label] = cached
|
161
|
+
|
162
|
+
check_ctx = CheckContext(
|
163
|
+
override=cached.override,
|
164
|
+
auth=cached.auth,
|
165
|
+
headers=cached.headers,
|
166
|
+
config=cached.config,
|
167
|
+
transport_kwargs=cached.transport_kwargs,
|
168
|
+
recorder=self.recorder,
|
169
|
+
)
|
142
170
|
validate_response(
|
143
171
|
response=response,
|
144
172
|
case=case,
|
145
173
|
stateful_ctx=ctx,
|
146
|
-
check_ctx=
|
147
|
-
checks=
|
174
|
+
check_ctx=check_ctx,
|
175
|
+
checks=check_ctx.checks,
|
148
176
|
control=engine.control,
|
149
177
|
recorder=self.recorder,
|
150
178
|
additional_checks=additional_checks,
|
@@ -169,7 +197,7 @@ def execute_state_machine_loop(
|
|
169
197
|
ctx.reset_scenario()
|
170
198
|
super().teardown()
|
171
199
|
|
172
|
-
seed = config.
|
200
|
+
seed = engine.config.seed
|
173
201
|
|
174
202
|
while True:
|
175
203
|
# This loop is running until no new failures are found in a single iteration
|
@@ -187,16 +215,13 @@ def execute_state_machine_loop(
|
|
187
215
|
)
|
188
216
|
break
|
189
217
|
suite_status = Status.SUCCESS
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
seed += 1
|
195
|
-
else:
|
196
|
-
InstrumentedStateMachine = _InstrumentedStateMachine
|
218
|
+
InstrumentedStateMachine = hypothesis.seed(seed)(_InstrumentedStateMachine)
|
219
|
+
# Predictably change the seed to avoid re-running the same sequences if tests fail
|
220
|
+
# yet have reproducible results
|
221
|
+
seed += 1
|
197
222
|
try:
|
198
223
|
with catch_warnings(), ignore_hypothesis_output(): # type: ignore
|
199
|
-
InstrumentedStateMachine.run(settings=
|
224
|
+
InstrumentedStateMachine.run(settings=hypothesis_settings)
|
200
225
|
except KeyboardInterrupt:
|
201
226
|
# Raised in the state machine when the stop event is set or it is raised by the user's code
|
202
227
|
# that is placed in the base class of the state machine.
|
@@ -220,7 +245,7 @@ def execute_state_machine_loop(
|
|
220
245
|
ctx.mark_as_seen_in_run(failure)
|
221
246
|
continue
|
222
247
|
except Flaky:
|
223
|
-
|
248
|
+
# Ignore flakiness
|
224
249
|
if engine.has_reached_the_failure_limit:
|
225
250
|
break # type: ignore[unreachable]
|
226
251
|
# Mark all failures in this suite as seen to prevent them being re-discovered
|
@@ -230,7 +255,7 @@ def execute_state_machine_loop(
|
|
230
255
|
if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
|
231
256
|
# Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
|
232
257
|
# values are possible to generate based on the previous observations, we retry the generation
|
233
|
-
if ctx.completed_scenarios >=
|
258
|
+
if ctx.completed_scenarios >= hypothesis_settings.max_examples:
|
234
259
|
# Avoid infinite restarts
|
235
260
|
break
|
236
261
|
continue
|
@@ -16,6 +16,7 @@ from schemathesis.core.result import Ok
|
|
16
16
|
from schemathesis.engine import Status, events
|
17
17
|
from schemathesis.engine.phases import PhaseName, PhaseSkipReason
|
18
18
|
from schemathesis.engine.recorder import ScenarioRecorder
|
19
|
+
from schemathesis.generation import overrides
|
19
20
|
from schemathesis.generation.hypothesis.builder import HypothesisTestConfig, HypothesisTestMode
|
20
21
|
from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
|
21
22
|
|
@@ -42,7 +43,6 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
|
|
42
43
|
else:
|
43
44
|
mode = HypothesisTestMode.FUZZING
|
44
45
|
producer = TaskProducer(engine)
|
45
|
-
workers_num = engine.config.execution.workers_num
|
46
46
|
|
47
47
|
suite_started = events.SuiteStarted(phase=phase.name)
|
48
48
|
|
@@ -53,7 +53,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
|
|
53
53
|
|
54
54
|
try:
|
55
55
|
with WorkerPool(
|
56
|
-
workers_num=
|
56
|
+
workers_num=engine.config.workers,
|
57
57
|
producer=producer,
|
58
58
|
worker_factory=worker_task,
|
59
59
|
ctx=engine,
|
@@ -160,16 +160,24 @@ def worker_task(
|
|
160
160
|
|
161
161
|
if isinstance(result, Ok):
|
162
162
|
operation = result.ok()
|
163
|
-
|
163
|
+
phases = ctx.config.phases_for(operation=operation)
|
164
|
+
# Skip tests if this phase is disabled
|
165
|
+
if (
|
166
|
+
(phase == PhaseName.EXAMPLES and not phases.examples.enabled)
|
167
|
+
or (phase == PhaseName.FUZZING and not phases.fuzzing.enabled)
|
168
|
+
or (phase == PhaseName.COVERAGE and not phases.coverage.enabled)
|
169
|
+
):
|
170
|
+
continue
|
171
|
+
as_strategy_kwargs = get_strategy_kwargs(ctx, operation=operation)
|
164
172
|
try:
|
165
173
|
test_function = create_test(
|
166
174
|
operation=operation,
|
167
175
|
test_func=test_func,
|
168
176
|
config=HypothesisTestConfig(
|
169
177
|
modes=[mode],
|
170
|
-
settings=ctx.config.
|
171
|
-
seed=ctx.config.
|
172
|
-
|
178
|
+
settings=ctx.config.get_hypothesis_settings(operation=operation, phase=phase.name),
|
179
|
+
seed=ctx.config.seed,
|
180
|
+
project=ctx.config,
|
173
181
|
as_strategy_kwargs=as_strategy_kwargs,
|
174
182
|
),
|
175
183
|
)
|
@@ -191,14 +199,14 @@ def worker_task(
|
|
191
199
|
events_queue.put(events.Interrupted(phase=phase))
|
192
200
|
|
193
201
|
|
194
|
-
def get_strategy_kwargs(ctx: EngineContext, operation: APIOperation) -> dict[str, Any]:
|
202
|
+
def get_strategy_kwargs(ctx: EngineContext, *, operation: APIOperation) -> dict[str, Any]:
|
195
203
|
kwargs = {}
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
}
|
204
|
+
override = overrides.for_operation(ctx.config, operation=operation)
|
205
|
+
for location in ("query", "headers", "cookies", "path_parameters"):
|
206
|
+
entry = getattr(override, location)
|
207
|
+
if entry:
|
208
|
+
kwargs[location] = entry
|
209
|
+
headers = ctx.config.headers_for(operation=operation)
|
210
|
+
if headers:
|
211
|
+
kwargs["headers"] = {key: value for key, value in headers.items() if key.lower() != "user-agent"}
|
204
212
|
return kwargs
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
3
3
|
import time
|
4
4
|
import unittest
|
5
5
|
import uuid
|
6
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Any, Callable
|
7
7
|
from warnings import WarningMessage, catch_warnings
|
8
8
|
|
9
9
|
import requests
|
@@ -11,8 +11,10 @@ from hypothesis.errors import InvalidArgument
|
|
11
11
|
from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
|
12
12
|
from jsonschema.exceptions import SchemaError as JsonSchemaError
|
13
13
|
from jsonschema.exceptions import ValidationError
|
14
|
+
from requests.structures import CaseInsensitiveDict
|
14
15
|
|
15
|
-
from schemathesis.checks import CheckContext,
|
16
|
+
from schemathesis.checks import CheckContext, run_checks
|
17
|
+
from schemathesis.config._generation import GenerationConfig
|
16
18
|
from schemathesis.core.compat import BaseExceptionGroup
|
17
19
|
from schemathesis.core.control import SkipTest
|
18
20
|
from schemathesis.core.errors import (
|
@@ -37,7 +39,7 @@ from schemathesis.engine.errors import (
|
|
37
39
|
)
|
38
40
|
from schemathesis.engine.phases import PhaseName
|
39
41
|
from schemathesis.engine.recorder import ScenarioRecorder
|
40
|
-
from schemathesis.generation import targets
|
42
|
+
from schemathesis.generation import overrides, targets
|
41
43
|
from schemathesis.generation.case import Case
|
42
44
|
from schemathesis.generation.hypothesis.builder import (
|
43
45
|
InvalidHeadersExampleMark,
|
@@ -85,10 +87,37 @@ def run_test(
|
|
85
87
|
is_final=False,
|
86
88
|
)
|
87
89
|
|
90
|
+
phase_name = phase.value.lower()
|
91
|
+
assert phase_name in ("examples", "coverage", "fuzzing", "stateful")
|
92
|
+
|
93
|
+
operation_config = ctx.config.operations.get_for_operation(operation)
|
94
|
+
continue_on_failure = operation_config.continue_on_failure or ctx.config.continue_on_failure or False
|
95
|
+
generation = ctx.config.generation_for(operation=operation, phase=phase_name)
|
96
|
+
override = overrides.for_operation(ctx.config, operation=operation)
|
97
|
+
auth = ctx.config.auth_for(operation=operation)
|
98
|
+
headers = ctx.config.headers_for(operation=operation)
|
99
|
+
transport_kwargs = ctx.get_transport_kwargs(operation=operation)
|
100
|
+
check_ctx = CheckContext(
|
101
|
+
override=override,
|
102
|
+
auth=auth,
|
103
|
+
headers=CaseInsensitiveDict(headers) if headers else None,
|
104
|
+
config=ctx.config.checks_config_for(operation=operation, phase=phase_name),
|
105
|
+
transport_kwargs=transport_kwargs,
|
106
|
+
recorder=recorder,
|
107
|
+
)
|
108
|
+
|
88
109
|
try:
|
89
110
|
setup_hypothesis_database_key(test_function, operation)
|
90
111
|
with catch_warnings(record=True) as warnings, ignore_hypothesis_output():
|
91
|
-
test_function(
|
112
|
+
test_function(
|
113
|
+
ctx=ctx,
|
114
|
+
errors=errors,
|
115
|
+
check_ctx=check_ctx,
|
116
|
+
recorder=recorder,
|
117
|
+
generation=generation,
|
118
|
+
transport_kwargs=transport_kwargs,
|
119
|
+
continue_on_failure=continue_on_failure,
|
120
|
+
)
|
92
121
|
# Test body was not executed at all - Hypothesis did not generate any tests, but there is no error
|
93
122
|
status = Status.SUCCESS
|
94
123
|
except (SkipTest, unittest.case.SkipTest) as exc:
|
@@ -147,6 +176,7 @@ def run_test(
|
|
147
176
|
exc,
|
148
177
|
path=operation.path,
|
149
178
|
method=operation.method,
|
179
|
+
config=ctx.config.output,
|
150
180
|
)
|
151
181
|
)
|
152
182
|
except HypothesisRefResolutionError:
|
@@ -180,7 +210,7 @@ def run_test(
|
|
180
210
|
yield non_fatal_error(exc)
|
181
211
|
if (
|
182
212
|
status == Status.SUCCESS
|
183
|
-
and
|
213
|
+
and continue_on_failure
|
184
214
|
and any(check.status == Status.FAILURE for checks in recorder.checks.values() for check in checks)
|
185
215
|
):
|
186
216
|
status = Status.FAILURE
|
@@ -237,25 +267,49 @@ def get_invalid_regular_expression_message(warnings: list[WarningMessage]) -> st
|
|
237
267
|
|
238
268
|
|
239
269
|
def cached_test_func(f: Callable) -> Callable:
|
240
|
-
def wrapped(
|
270
|
+
def wrapped(
|
271
|
+
*,
|
272
|
+
ctx: EngineContext,
|
273
|
+
case: Case,
|
274
|
+
errors: list[Exception],
|
275
|
+
check_ctx: CheckContext,
|
276
|
+
recorder: ScenarioRecorder,
|
277
|
+
generation: GenerationConfig,
|
278
|
+
transport_kwargs: dict[str, Any],
|
279
|
+
continue_on_failure: bool,
|
280
|
+
) -> None:
|
241
281
|
try:
|
242
282
|
if ctx.has_to_stop:
|
243
283
|
raise KeyboardInterrupt
|
244
|
-
if
|
284
|
+
if generation.unique_inputs:
|
245
285
|
cached = ctx.get_cached_outcome(case)
|
246
286
|
if isinstance(cached, BaseException):
|
247
287
|
raise cached
|
248
288
|
elif cached is None:
|
249
289
|
return None
|
250
290
|
try:
|
251
|
-
f(
|
291
|
+
f(
|
292
|
+
case=case,
|
293
|
+
check_ctx=check_ctx,
|
294
|
+
recorder=recorder,
|
295
|
+
generation=generation,
|
296
|
+
transport_kwargs=transport_kwargs,
|
297
|
+
continue_on_failure=continue_on_failure,
|
298
|
+
)
|
252
299
|
except BaseException as exc:
|
253
300
|
ctx.cache_outcome(case, exc)
|
254
301
|
raise
|
255
302
|
else:
|
256
303
|
ctx.cache_outcome(case, None)
|
257
304
|
else:
|
258
|
-
f(
|
305
|
+
f(
|
306
|
+
case=case,
|
307
|
+
check_ctx=check_ctx,
|
308
|
+
recorder=recorder,
|
309
|
+
generation=generation,
|
310
|
+
transport_kwargs=transport_kwargs,
|
311
|
+
continue_on_failure=continue_on_failure,
|
312
|
+
)
|
259
313
|
except (KeyboardInterrupt, Failure):
|
260
314
|
raise
|
261
315
|
except Exception as exc:
|
@@ -268,10 +322,18 @@ def cached_test_func(f: Callable) -> Callable:
|
|
268
322
|
|
269
323
|
|
270
324
|
@cached_test_func
|
271
|
-
def test_func(
|
325
|
+
def test_func(
|
326
|
+
*,
|
327
|
+
case: Case,
|
328
|
+
check_ctx: CheckContext,
|
329
|
+
recorder: ScenarioRecorder,
|
330
|
+
generation: GenerationConfig,
|
331
|
+
transport_kwargs: dict[str, Any],
|
332
|
+
continue_on_failure: bool,
|
333
|
+
) -> None:
|
272
334
|
recorder.record_case(parent_id=None, transition=None, case=case)
|
273
335
|
try:
|
274
|
-
response = case.call(**
|
336
|
+
response = case.call(**transport_kwargs)
|
275
337
|
except (requests.Timeout, requests.ConnectionError) as error:
|
276
338
|
if isinstance(error.request, requests.Request):
|
277
339
|
recorder.record_request(case_id=case.id, request=error.request.prepare())
|
@@ -279,13 +341,12 @@ def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) ->
|
|
279
341
|
recorder.record_request(case_id=case.id, request=error.request)
|
280
342
|
raise
|
281
343
|
recorder.record_response(case_id=case.id, response=response)
|
282
|
-
targets.run(
|
344
|
+
targets.run(generation.maximize, case=case, response=response)
|
283
345
|
validate_response(
|
284
346
|
case=case,
|
285
|
-
ctx=
|
286
|
-
checks=ctx.config.execution.checks,
|
347
|
+
ctx=check_ctx,
|
287
348
|
response=response,
|
288
|
-
continue_on_failure=
|
349
|
+
continue_on_failure=continue_on_failure,
|
289
350
|
recorder=recorder,
|
290
351
|
)
|
291
352
|
|
@@ -294,7 +355,6 @@ def validate_response(
|
|
294
355
|
*,
|
295
356
|
case: Case,
|
296
357
|
ctx: CheckContext,
|
297
|
-
checks: Iterable[CheckFunction],
|
298
358
|
response: Response,
|
299
359
|
continue_on_failure: bool,
|
300
360
|
recorder: ScenarioRecorder,
|
@@ -318,7 +378,7 @@ def validate_response(
|
|
318
378
|
case=case,
|
319
379
|
response=response,
|
320
380
|
ctx=ctx,
|
321
|
-
checks=checks,
|
381
|
+
checks=ctx.checks,
|
322
382
|
on_failure=on_failure,
|
323
383
|
on_success=on_success,
|
324
384
|
)
|
@@ -18,7 +18,7 @@ class TaskProducer:
|
|
18
18
|
"""Produces test tasks for workers to execute."""
|
19
19
|
|
20
20
|
def __init__(self, ctx: EngineContext) -> None:
|
21
|
-
self.operations = ctx.schema.get_all_operations(
|
21
|
+
self.operations = ctx.schema.get_all_operations()
|
22
22
|
self.lock = threading.Lock()
|
23
23
|
|
24
24
|
def next_operation(self) -> Result | None:
|
schemathesis/errors.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
"""Public Schemathesis errors."""
|
2
2
|
|
3
3
|
from schemathesis.core.errors import (
|
4
|
+
HookError,
|
4
5
|
IncorrectUsage,
|
5
6
|
InternalError,
|
6
7
|
InvalidHeadersExample,
|
@@ -21,6 +22,7 @@ from schemathesis.core.errors import (
|
|
21
22
|
)
|
22
23
|
|
23
24
|
__all__ = [
|
25
|
+
"HookError",
|
24
26
|
"IncorrectUsage",
|
25
27
|
"InternalError",
|
26
28
|
"InvalidHeadersExample",
|
schemathesis/filters.py
CHANGED
@@ -150,9 +150,8 @@ class FilterSet:
|
|
150
150
|
def clone(self) -> FilterSet:
|
151
151
|
return FilterSet(_includes=self._includes.copy(), _excludes=self._excludes.copy())
|
152
152
|
|
153
|
-
def
|
154
|
-
|
155
|
-
return [operation for operation in operations if self.match(SimpleNamespace(operation=operation))]
|
153
|
+
def applies_to(self, operation: APIOperation) -> bool:
|
154
|
+
return self.match(SimpleNamespace(operation=operation))
|
156
155
|
|
157
156
|
def match(self, ctx: HasAPIOperation) -> bool:
|
158
157
|
"""Determines whether the given operation should be included based on the defined filters.
|
@@ -1,16 +1,16 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
import random
|
4
|
-
from dataclasses import dataclass, field
|
5
|
-
from typing import TYPE_CHECKING
|
6
4
|
|
7
|
-
from schemathesis.generation.modes import GenerationMode
|
5
|
+
from schemathesis.generation.modes import GenerationMode
|
8
6
|
|
9
|
-
|
10
|
-
|
7
|
+
__all__ = [
|
8
|
+
"GenerationMode",
|
9
|
+
"generate_random_case_id",
|
10
|
+
]
|
11
11
|
|
12
12
|
|
13
|
-
DEFAULT_GENERATOR_MODES =
|
13
|
+
DEFAULT_GENERATOR_MODES = [GenerationMode.default()]
|
14
14
|
|
15
15
|
|
16
16
|
CASE_ID_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
@@ -26,28 +26,3 @@ def generate_random_case_id(length: int = 6) -> str:
|
|
26
26
|
number, rem = divmod(number, BASE)
|
27
27
|
output += CASE_ID_ALPHABET[rem]
|
28
28
|
return output
|
29
|
-
|
30
|
-
|
31
|
-
@dataclass
|
32
|
-
class HeaderConfig:
|
33
|
-
"""Configuration for generating headers."""
|
34
|
-
|
35
|
-
strategy: SearchStrategy[str] | None = None
|
36
|
-
|
37
|
-
|
38
|
-
@dataclass
|
39
|
-
class GenerationConfig:
|
40
|
-
"""Holds various configuration options relevant for data generation."""
|
41
|
-
|
42
|
-
modes: list[GenerationMode] = field(default_factory=lambda: [GenerationMode.default()])
|
43
|
-
# Allow generating `\x00` bytes in strings
|
44
|
-
allow_x00: bool = True
|
45
|
-
# Allowing using `null` for optional arguments in GraphQL queries
|
46
|
-
graphql_allow_null: bool = True
|
47
|
-
# Generate strings using the given codec
|
48
|
-
codec: str | None = "utf-8"
|
49
|
-
# Whether to generate security parameters
|
50
|
-
with_security_parameters: bool = True
|
51
|
-
# Header generation configuration
|
52
|
-
headers: HeaderConfig = field(default_factory=HeaderConfig)
|
53
|
-
unexpected_methods: set[str] | None = None
|