pytest-httpchain 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/PKG-INFO +10 -4
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/README.md +1 -1
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/pyproject.toml +9 -14
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/__init__.py +7 -0
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/carrier.py +223 -117
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/cli.py +60 -36
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/constants.py +3 -2
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/dataflow.py +25 -31
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/har_writer.py +23 -4
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/plugin.py +75 -36
- pytest_httpchain-0.7.0/src/pytest_httpchain/report_formatter.py +88 -0
- pytest_httpchain-0.7.0/src/pytest_httpchain/schema.py +122 -0
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/utils.py +38 -0
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/validation.py +238 -111
- pytest_httpchain-0.5.0/src/pytest_httpchain/report_formatter.py +0 -69
- pytest_httpchain-0.5.0/src/pytest_httpchain/schema.py +0 -69
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/LICENSE +0 -0
- {pytest_httpchain-0.5.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/exceptions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pytest-httpchain
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: pytest plugin for HTTP testing using JSON files
|
|
5
5
|
Keywords: testing,pytest,requests
|
|
6
6
|
Author: Alexander Eresov
|
|
@@ -15,14 +15,20 @@ Classifier: Programming Language :: Python :: 3
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
16
|
Classifier: Topic :: Software Development :: Testing
|
|
17
17
|
Requires-Dist: httpx[http2]>=0.27.0
|
|
18
|
+
Requires-Dist: jmespath>=1.0.0
|
|
19
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
18
20
|
Requires-Dist: pydantic>=2.11.7
|
|
19
|
-
Requires-Dist: pyrate-limiter>=
|
|
21
|
+
Requires-Dist: pyrate-limiter>=4.0.0
|
|
22
|
+
Requires-Dist: pytest>=8.4
|
|
23
|
+
Requires-Dist: pytest-httpchain-core
|
|
20
24
|
Requires-Dist: pytest-httpchain-jsonref
|
|
21
25
|
Requires-Dist: pytest-httpchain-models
|
|
22
26
|
Requires-Dist: pytest-httpchain-templates
|
|
27
|
+
Requires-Dist: pytest-httpchain-userfunc
|
|
23
28
|
Requires-Dist: pytest-order>=1.3.0
|
|
29
|
+
Requires-Dist: simpleeval>=1.0.3
|
|
24
30
|
Requires-Dist: typer>=0.16.0
|
|
25
|
-
Requires-Python: >=3.13
|
|
31
|
+
Requires-Python: >=3.13
|
|
26
32
|
Description-Content-Type: text/markdown
|
|
27
33
|
|
|
28
34
|
[](https://pypi.python.org/pypi/pytest-httpchain)
|
|
@@ -71,7 +77,7 @@ A key-value store persists throughout scenario execution. Variables, fixtures, a
|
|
|
71
77
|
|
|
72
78
|
- **JMESPath** — Extract values from JSON responses directly
|
|
73
79
|
- **JSON Schema** — Validate response structure against a schema
|
|
74
|
-
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://
|
|
80
|
+
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://www.python-httpx.org/advanced/authentication/#custom-authentication-schemes)
|
|
75
81
|
|
|
76
82
|
### Full pytest integration
|
|
77
83
|
|
|
@@ -44,7 +44,7 @@ A key-value store persists throughout scenario execution. Variables, fixtures, a
|
|
|
44
44
|
|
|
45
45
|
- **JMESPath** — Extract values from JSON responses directly
|
|
46
46
|
- **JSON Schema** — Validate response structure against a schema
|
|
47
|
-
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://
|
|
47
|
+
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://www.python-httpx.org/advanced/authentication/#custom-authentication-schemes)
|
|
48
48
|
|
|
49
49
|
### Full pytest integration
|
|
50
50
|
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pytest-httpchain"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.0"
|
|
4
4
|
description = "pytest plugin for HTTP testing using JSON files"
|
|
5
5
|
readme = "README.md"
|
|
6
|
-
requires-python = ">=3.13
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
7
|
authors = [{ name = "Alexander Eresov", email = "aeresov@gmail.com" }]
|
|
8
8
|
dependencies = [
|
|
9
9
|
"httpx[http2]>=0.27.0",
|
|
10
|
+
"jmespath>=1.0.0",
|
|
11
|
+
"jsonschema>=4.0.0",
|
|
10
12
|
"pydantic>=2.11.7",
|
|
11
|
-
"pyrate-limiter>=
|
|
13
|
+
"pyrate-limiter>=4.0.0",
|
|
14
|
+
"pytest>=8.4",
|
|
15
|
+
"pytest-httpchain-core",
|
|
12
16
|
"pytest-httpchain-jsonref",
|
|
13
17
|
"pytest-httpchain-models",
|
|
14
18
|
"pytest-httpchain-templates",
|
|
19
|
+
"pytest-httpchain-userfunc",
|
|
15
20
|
"pytest-order>=1.3.0",
|
|
21
|
+
"simpleeval>=1.0.3",
|
|
16
22
|
"typer>=0.16.0",
|
|
17
23
|
]
|
|
18
24
|
keywords = ["testing", "pytest", "requests"]
|
|
@@ -40,8 +46,6 @@ dev = [
|
|
|
40
46
|
docs = [
|
|
41
47
|
"mkdocs>=1.6.1",
|
|
42
48
|
"mkdocs-material>=9.6.16",
|
|
43
|
-
"mkdocstrings>=0.30.0",
|
|
44
|
-
"mkdocstrings-python>=1.16.12",
|
|
45
49
|
]
|
|
46
50
|
|
|
47
51
|
[tool.uv]
|
|
@@ -107,15 +111,6 @@ pythonpath = [
|
|
|
107
111
|
]
|
|
108
112
|
testpaths = ["tests", "packages/*/tests"]
|
|
109
113
|
|
|
110
|
-
[tool.pyright]
|
|
111
|
-
exclude = [".tox/"]
|
|
112
|
-
ignore = [".venv/"]
|
|
113
|
-
reportMissingImports = "error"
|
|
114
|
-
reportMissingTypeStubs = false
|
|
115
|
-
reportExplicitAny = false
|
|
116
|
-
typeCheckingMode = "off"
|
|
117
|
-
pythonPlatform = "Linux"
|
|
118
|
-
|
|
119
114
|
[tool.coverage.run]
|
|
120
115
|
source = [
|
|
121
116
|
"src",
|
|
@@ -21,3 +21,10 @@ Example test file (test_api.http.json):
|
|
|
21
21
|
]
|
|
22
22
|
}
|
|
23
23
|
"""
|
|
24
|
+
|
|
25
|
+
# Re-export the user-facing warning so consumers can filter it without reaching
|
|
26
|
+
# into the plugin module. The definition stays in plugin.py because the pytest11
|
|
27
|
+
# entry point and the collection hook reference it there directly.
|
|
28
|
+
from pytest_httpchain.plugin import ScenarioValidationWarning
|
|
29
|
+
|
|
30
|
+
__all__ = ["ScenarioValidationWarning"]
|
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
"""Test execution engine for pytest-httpchain scenarios.
|
|
2
|
+
|
|
3
|
+
``create_test_class`` turns a validated :class:`Scenario` into a dynamic pytest
|
|
4
|
+
test class (a subclass of :class:`Carrier`), one ``test_NN - <stage name>``
|
|
5
|
+
method per stage, ordered by the ``order(i)`` marker so the stages run as a
|
|
6
|
+
chain. Each scenario gets its own subclass; the per-scenario mutable state
|
|
7
|
+
(``client``, ``global_context``, ``aborted``, ``last_request``/``last_response``,
|
|
8
|
+
``active_context_managers``) lives at the *class* level and is overridden in the
|
|
9
|
+
subclass dict, so the stage methods — which are classmethods operating on
|
|
10
|
+
``cls`` — share one running context across the chain while different scenarios
|
|
11
|
+
stay isolated from each other.
|
|
12
|
+
|
|
13
|
+
Per stage, :meth:`Carrier.execute_stage` gates on the abort/``always_run`` flow,
|
|
14
|
+
layers fixtures and substitutions over the global context, expands any
|
|
15
|
+
``parallel`` config into iterations (sequential or thread-pooled, optionally
|
|
16
|
+
rate limited), executes the HTTP request via httpx, runs the response
|
|
17
|
+
verify/save steps, and on full success commits the collected saves as a new
|
|
18
|
+
global-context layer for later stages. Expected failures (bad scenario, failed
|
|
19
|
+
verification, unreachable server) are surfaced via ``pytest.fail(pytrace=False)``
|
|
20
|
+
so the report stays clean rather than dumping an internal traceback.
|
|
21
|
+
"""
|
|
22
|
+
|
|
1
23
|
import base64
|
|
2
24
|
import inspect
|
|
3
25
|
import json
|
|
@@ -17,7 +39,7 @@ import jmespath
|
|
|
17
39
|
import jmespath.exceptions
|
|
18
40
|
import jsonschema
|
|
19
41
|
import pytest
|
|
20
|
-
from pydantic import
|
|
42
|
+
from pydantic import ValidationError
|
|
21
43
|
from pyrate_limiter import Duration, Limiter, Rate
|
|
22
44
|
from pytest_httpchain_models import (
|
|
23
45
|
Base64Body,
|
|
@@ -53,6 +75,12 @@ from .utils import call_user_function, make_marker, process_substitutions
|
|
|
53
75
|
|
|
54
76
|
logger = logging.getLogger(__name__)
|
|
55
77
|
|
|
78
|
+
# Exceptions that represent an expected stage failure (a bad scenario, a failed
|
|
79
|
+
# verification, an unreachable server) rather than a bug in the plugin. Both the
|
|
80
|
+
# sequential and the parallel execution paths catch these and turn them into a
|
|
81
|
+
# clean pytest failure instead of a raw traceback.
|
|
82
|
+
_STAGE_FAILURE_EXCEPTIONS = (StageExecutionError, TemplatesError, ValidationError)
|
|
83
|
+
|
|
56
84
|
|
|
57
85
|
@dataclass
|
|
58
86
|
class ParallelIterationResult:
|
|
@@ -78,19 +106,41 @@ class Carrier:
|
|
|
78
106
|
active_context_managers: ClassVar[list[AbstractContextManager]] = []
|
|
79
107
|
max_parallel_iterations: ClassVar[int] = 10_000
|
|
80
108
|
|
|
109
|
+
@classmethod
|
|
110
|
+
def _resolve_always_run(cls, stage: Stage, stage_fixtures: dict[str, Any]) -> bool:
|
|
111
|
+
"""Resolve ``always_run``, evaluating a template form against the context
|
|
112
|
+
available at stage start: fixtures and parametrize parameters plus the
|
|
113
|
+
global context (scenario substitutions and earlier saves). Stage
|
|
114
|
+
substitutions are not yet processed at this point. The result is coerced
|
|
115
|
+
with Python truthiness."""
|
|
116
|
+
if isinstance(stage.always_run, bool):
|
|
117
|
+
return stage.always_run
|
|
118
|
+
try:
|
|
119
|
+
return bool(walk(stage.always_run, ChainMap(stage_fixtures, cls.global_context)))
|
|
120
|
+
except TemplatesError as e:
|
|
121
|
+
raise StageExecutionError(f"Failed to evaluate always_run template: {e}") from e
|
|
122
|
+
|
|
81
123
|
@classmethod
|
|
82
124
|
def execute_stage(cls, stage: Stage, fixture_kwargs: dict[str, Any]) -> None:
|
|
125
|
+
"""Execute one stage end to end.
|
|
126
|
+
|
|
127
|
+
Gates on the abort/``always_run`` flow, layers the stage context
|
|
128
|
+
(fixtures + substitutions over the global context), builds the iteration
|
|
129
|
+
matrix (``_build_iteration_substitutions``), runs the iterations
|
|
130
|
+
sequentially or across a thread pool (``_run_iterations``), and on full
|
|
131
|
+
success commits the collected saves as a new global-context layer.
|
|
132
|
+
|
|
133
|
+
Any expected failure is reported via ``pytest.fail`` (clean, no
|
|
134
|
+
traceback) and marks the chain aborted so later stages skip unless they
|
|
135
|
+
opt into ``always_run``. A failing stage commits **no** saves, so the
|
|
136
|
+
global context is left unchanged (deterministic) rather than carrying a
|
|
137
|
+
thread-timing-dependent subset of a parallel run.
|
|
138
|
+
"""
|
|
83
139
|
try:
|
|
84
|
-
|
|
85
|
-
pytest.skip(reason="Flow aborted")
|
|
140
|
+
stage_fixtures = cls._build_stage_fixtures(fixture_kwargs)
|
|
86
141
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
for name, value in fixture_kwargs.items():
|
|
90
|
-
if callable(value) and not inspect.isclass(value):
|
|
91
|
-
stage_fixtures[name] = cls._wrap_factory_fixture(value)
|
|
92
|
-
else:
|
|
93
|
-
stage_fixtures[name] = value
|
|
142
|
+
if cls.aborted and not cls._resolve_always_run(stage, stage_fixtures):
|
|
143
|
+
pytest.skip(reason="Flow aborted")
|
|
94
144
|
|
|
95
145
|
# Build base context for iterations (substitutions + fixtures + global)
|
|
96
146
|
local_context = ChainMap(stage_fixtures, cls.global_context)
|
|
@@ -100,99 +150,145 @@ class Carrier:
|
|
|
100
150
|
logger.info(f"global context on start: {json.dumps(dict(cls.global_context), indent=2, default=str)}")
|
|
101
151
|
logger.info(f"local context on start: {json.dumps(dict(local_context), indent=2, default=str)}")
|
|
102
152
|
|
|
103
|
-
# build iterations
|
|
104
|
-
iteration_substitutions: list[dict[str, Any]] = [{}]
|
|
105
153
|
parallel_config = walk(stage.parallel, local_context) if stage.parallel else None
|
|
106
|
-
|
|
107
|
-
case None:
|
|
108
|
-
pass
|
|
109
|
-
case ParallelRepeatConfig(repeat=repeat_count):
|
|
110
|
-
iteration_substitutions = [{} for _ in range(repeat_count)]
|
|
111
|
-
case ParallelForeachConfig(foreach=foreach_steps):
|
|
112
|
-
# same algorithm like what pytest does for parametrize marker
|
|
113
|
-
for step in foreach_steps:
|
|
114
|
-
match step:
|
|
115
|
-
case IndividualParameter(individual=individual):
|
|
116
|
-
param_name = next(iter(individual.keys()))
|
|
117
|
-
values = individual[param_name]
|
|
118
|
-
iteration_substitutions = [{**existing, param_name: val} for val in values for existing in iteration_substitutions]
|
|
119
|
-
case CombinationsParameter(combinations=combinations):
|
|
120
|
-
combos: list[dict[str, Any] | SimpleNamespace] = [vars(item) if isinstance(item, SimpleNamespace) else item for item in combinations]
|
|
121
|
-
iteration_substitutions = [{**existing, **combo} for combo in combos for existing in iteration_substitutions]
|
|
122
|
-
|
|
123
|
-
if len(iteration_substitutions) > cls.max_parallel_iterations:
|
|
124
|
-
raise StageExecutionError(
|
|
125
|
-
f"Parallel iteration count ({len(iteration_substitutions)}) exceeds maximum ({cls.max_parallel_iterations}). "
|
|
126
|
-
f"Set 'max_parallel_iterations' in pytest.ini to increase the limit."
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
# execute iterations
|
|
130
|
-
max_concurrency = parallel_config.max_concurrency if parallel_config else 1
|
|
131
|
-
calls_per_sec = parallel_config.calls_per_sec if parallel_config else None
|
|
132
|
-
max_rate_limit_delay = parallel_config.max_rate_limit_delay if parallel_config else 60
|
|
154
|
+
iteration_substitutions = cls._build_iteration_substitutions(parallel_config)
|
|
133
155
|
|
|
134
156
|
total = len(iteration_substitutions)
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
157
|
+
if total == 0:
|
|
158
|
+
# Unreachable via validated models (foreach values have min_length=1,
|
|
159
|
+
# repeat is PositiveInt), but guard so a future gap fails loudly
|
|
160
|
+
# instead of passing a stage that never sent a request.
|
|
161
|
+
raise StageExecutionError("Parallel configuration produced zero iterations; foreach/repeat must yield at least one item")
|
|
162
|
+
if total > cls.max_parallel_iterations:
|
|
163
|
+
raise StageExecutionError(
|
|
164
|
+
f"Parallel iteration count ({total}) exceeds maximum ({cls.max_parallel_iterations}). Set 'max_parallel_iterations' in pytest.ini to increase the limit."
|
|
165
|
+
)
|
|
138
166
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
167
|
+
results, first_error = cls._run_iterations(stage, local_context, iteration_substitutions, parallel_config)
|
|
168
|
+
|
|
169
|
+
if first_error is None:
|
|
170
|
+
# Commit saves only on full success. A failed (parallel) stage must
|
|
171
|
+
# leave the global context untouched, not commit a non-deterministic
|
|
172
|
+
# subset of iterations whose saves happened to land before the error.
|
|
173
|
+
all_saves: dict[str, Any] = {}
|
|
174
|
+
for iter_result in results:
|
|
175
|
+
if iter_result is not None:
|
|
176
|
+
all_saves.update(iter_result.saved_context)
|
|
177
|
+
cls.last_request = iter_result.request
|
|
178
|
+
cls.last_response = iter_result.response
|
|
179
|
+
logger.info(f"updates for global context: {json.dumps(all_saves, indent=2, default=str)}")
|
|
180
|
+
cls.global_context = cls.global_context.new_child(all_saves)
|
|
144
181
|
else:
|
|
145
|
-
workers = min(max_concurrency, total) if total > 0 else 1
|
|
146
|
-
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
147
|
-
futures: dict[Future[ParallelIterationResult], int] = {}
|
|
148
|
-
for idx, iter_vars in enumerate(iteration_substitutions):
|
|
149
|
-
future = executor.submit(cls._execute_single_iteration, stage, local_context, iter_vars, limiter)
|
|
150
|
-
futures[future] = idx
|
|
151
|
-
|
|
152
|
-
for future in as_completed(futures):
|
|
153
|
-
idx = futures[future]
|
|
154
|
-
try:
|
|
155
|
-
results[idx] = future.result()
|
|
156
|
-
except (StageExecutionError, TemplatesError, ValidationError) as e:
|
|
157
|
-
first_error = (idx, e)
|
|
158
|
-
executor.shutdown(wait=False, cancel_futures=True)
|
|
159
|
-
break
|
|
160
|
-
|
|
161
|
-
# Apply results in index order - collect all saves for new layer
|
|
162
|
-
all_saves: dict[str, Any] = {}
|
|
163
|
-
for iter_result in results:
|
|
164
|
-
if iter_result is not None:
|
|
165
|
-
all_saves.update(iter_result.saved_context)
|
|
166
|
-
cls.last_request = iter_result.request
|
|
167
|
-
cls.last_response = iter_result.response
|
|
168
|
-
|
|
169
|
-
logger.info(f"updates for global context: {json.dumps(all_saves, indent=2, default=str)}")
|
|
170
|
-
|
|
171
|
-
# Add response saves as new layer
|
|
172
|
-
cls.global_context = cls.global_context.new_child(all_saves)
|
|
173
|
-
|
|
174
|
-
# Handle error
|
|
175
|
-
if first_error:
|
|
176
182
|
idx, exc = first_error
|
|
177
|
-
#
|
|
183
|
+
# Surface the failed iteration's request/response in the report.
|
|
178
184
|
if isinstance(exc, StageExecutionError):
|
|
179
185
|
if exc.request is not None:
|
|
180
186
|
cls.last_request = exc.request
|
|
181
187
|
if exc.response is not None:
|
|
182
188
|
cls.last_response = exc.response
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
189
|
+
# Only label the failure as parallel when the user configured
|
|
190
|
+
# `parallel`; otherwise re-raise the original error unchanged so a
|
|
191
|
+
# plain stage failure isn't misreported as "Parallel execution failed".
|
|
192
|
+
if parallel_config is not None:
|
|
193
|
+
raise StageExecutionError(f"Parallel execution failed at iteration {idx}: {exc}") from exc
|
|
194
|
+
raise exc
|
|
195
|
+
|
|
196
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
197
|
+
# Detect xfail structurally (marker name == "xfail") rather than by a
|
|
198
|
+
# substring scan of the raw mark string, so e.g. `skip(reason="...xfail...")`
|
|
199
|
+
# or a custom `my_xfail` marker is not misclassified. Every mark already
|
|
200
|
+
# round-tripped through make_marker() at collection time (create_test_class),
|
|
201
|
+
# so parsing here cannot raise on a previously-validated scenario.
|
|
202
|
+
is_xfail = any(make_marker(mark).name == "xfail" for mark in stage.marks)
|
|
191
203
|
if not is_xfail:
|
|
192
204
|
logger.error(str(e))
|
|
193
205
|
cls.aborted = True
|
|
194
206
|
pytest.fail(reason=str(e), pytrace=False)
|
|
195
207
|
|
|
208
|
+
@classmethod
|
|
209
|
+
def _build_stage_fixtures(cls, fixture_kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
210
|
+
"""Resolve injected fixtures, wrapping callable (factory) fixtures so they
|
|
211
|
+
can be invoked from template expressions while plain values pass through."""
|
|
212
|
+
stage_fixtures: dict[str, Any] = {}
|
|
213
|
+
for name, value in fixture_kwargs.items():
|
|
214
|
+
if callable(value) and not inspect.isclass(value):
|
|
215
|
+
stage_fixtures[name] = cls._wrap_factory_fixture(value)
|
|
216
|
+
else:
|
|
217
|
+
stage_fixtures[name] = value
|
|
218
|
+
return stage_fixtures
|
|
219
|
+
|
|
220
|
+
@staticmethod
|
|
221
|
+
def _build_iteration_substitutions(parallel_config: Any) -> list[dict[str, Any]]:
|
|
222
|
+
"""Expand the (already template-resolved) parallel config into per-iteration
|
|
223
|
+
substitution dicts: non-parallel -> a single empty dict; ``repeat`` -> N
|
|
224
|
+
empties; ``foreach`` -> the cross-product of its parameter steps (the same
|
|
225
|
+
algorithm pytest uses for the ``parametrize`` marker)."""
|
|
226
|
+
iteration_substitutions: list[dict[str, Any]] = [{}]
|
|
227
|
+
match parallel_config:
|
|
228
|
+
case None:
|
|
229
|
+
pass
|
|
230
|
+
case ParallelRepeatConfig(repeat=repeat_count):
|
|
231
|
+
iteration_substitutions = [{} for _ in range(repeat_count)]
|
|
232
|
+
case ParallelForeachConfig(foreach=foreach_steps):
|
|
233
|
+
for step in foreach_steps:
|
|
234
|
+
match step:
|
|
235
|
+
case IndividualParameter(individual=individual):
|
|
236
|
+
param_name = next(iter(individual.keys()))
|
|
237
|
+
values = individual[param_name]
|
|
238
|
+
iteration_substitutions = [{**existing, param_name: val} for val in values for existing in iteration_substitutions]
|
|
239
|
+
case CombinationsParameter(combinations=combinations):
|
|
240
|
+
combos: list[dict[str, Any] | SimpleNamespace] = [vars(item) if isinstance(item, SimpleNamespace) else item for item in combinations]
|
|
241
|
+
iteration_substitutions = [{**existing, **combo} for combo in combos for existing in iteration_substitutions]
|
|
242
|
+
return iteration_substitutions
|
|
243
|
+
|
|
244
|
+
@classmethod
|
|
245
|
+
def _run_iterations(
|
|
246
|
+
cls,
|
|
247
|
+
stage: Stage,
|
|
248
|
+
local_context: ChainMap[str, Any],
|
|
249
|
+
iteration_substitutions: list[dict[str, Any]],
|
|
250
|
+
parallel_config: Any,
|
|
251
|
+
) -> tuple[list[ParallelIterationResult | None], tuple[int, Exception] | None]:
|
|
252
|
+
"""Run the iterations and return ``(results_by_index, first_error)``.
|
|
253
|
+
|
|
254
|
+
A single iteration runs inline; multiple iterations run in a
|
|
255
|
+
``ThreadPoolExecutor`` capped at ``max_concurrency``, with an optional
|
|
256
|
+
global rate limiter. On the first expected failure the pool is cancelled
|
|
257
|
+
and ``(index, exception)`` is returned; otherwise ``first_error`` is None.
|
|
258
|
+
"""
|
|
259
|
+
max_concurrency = parallel_config.max_concurrency if parallel_config else 1
|
|
260
|
+
calls_per_sec = parallel_config.calls_per_sec if parallel_config else None
|
|
261
|
+
max_rate_limit_delay = parallel_config.max_rate_limit_delay if parallel_config else 60
|
|
262
|
+
|
|
263
|
+
total = len(iteration_substitutions)
|
|
264
|
+
results: list[ParallelIterationResult | None] = [None] * total
|
|
265
|
+
first_error: tuple[int, Exception] | None = None
|
|
266
|
+
limiter = Limiter(Rate(calls_per_sec, Duration.SECOND)) if calls_per_sec else None
|
|
267
|
+
|
|
268
|
+
if total == 1:
|
|
269
|
+
try:
|
|
270
|
+
results[0] = cls._execute_single_iteration(stage, local_context, iteration_substitutions[0], limiter, max_rate_limit_delay)
|
|
271
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
272
|
+
first_error = (0, e)
|
|
273
|
+
else:
|
|
274
|
+
workers = min(max_concurrency, total)
|
|
275
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
276
|
+
futures: dict[Future[ParallelIterationResult], int] = {}
|
|
277
|
+
for idx, iter_vars in enumerate(iteration_substitutions):
|
|
278
|
+
future = executor.submit(cls._execute_single_iteration, stage, local_context, iter_vars, limiter, max_rate_limit_delay)
|
|
279
|
+
futures[future] = idx
|
|
280
|
+
|
|
281
|
+
for future in as_completed(futures):
|
|
282
|
+
idx = futures[future]
|
|
283
|
+
try:
|
|
284
|
+
results[idx] = future.result()
|
|
285
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
286
|
+
first_error = (idx, e)
|
|
287
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
288
|
+
break
|
|
289
|
+
|
|
290
|
+
return results, first_error
|
|
291
|
+
|
|
196
292
|
@staticmethod
|
|
197
293
|
def _build_request_kwargs(request_model: Request) -> dict[str, Any]:
|
|
198
294
|
request_kwargs: dict[str, Any] = {
|
|
@@ -209,7 +305,7 @@ class Carrier:
|
|
|
209
305
|
auth_result = call_user_function(request_model.auth)
|
|
210
306
|
request_kwargs["auth"] = auth_result
|
|
211
307
|
except UserFunctionError as e:
|
|
212
|
-
raise RequestError(f"Failed to configure authentication: {e}") from
|
|
308
|
+
raise RequestError(f"Failed to configure authentication: {e}") from e
|
|
213
309
|
|
|
214
310
|
match request_model.body:
|
|
215
311
|
case None:
|
|
@@ -234,8 +330,10 @@ class Carrier:
|
|
|
234
330
|
case BinaryBody(binary=file_path):
|
|
235
331
|
try:
|
|
236
332
|
request_kwargs["content"] = Path(file_path).read_bytes()
|
|
237
|
-
except FileNotFoundError:
|
|
238
|
-
raise RequestError(f"Binary file not found: {file_path}") from
|
|
333
|
+
except FileNotFoundError as e:
|
|
334
|
+
raise RequestError(f"Binary file not found: {file_path}") from e
|
|
335
|
+
except OSError as e:
|
|
336
|
+
raise RequestError(f"Cannot read binary file '{file_path}': {e}") from e
|
|
239
337
|
|
|
240
338
|
case FilesBody(files=file_paths):
|
|
241
339
|
files_list = []
|
|
@@ -243,8 +341,10 @@ class Carrier:
|
|
|
243
341
|
path = Path(file_path)
|
|
244
342
|
try:
|
|
245
343
|
files_list.append((field_name, (path.name, path.read_bytes())))
|
|
246
|
-
except FileNotFoundError:
|
|
247
|
-
raise RequestError(f"File not found for upload: {file_path}") from
|
|
344
|
+
except FileNotFoundError as e:
|
|
345
|
+
raise RequestError(f"File not found for upload: {file_path}") from e
|
|
346
|
+
except OSError as e:
|
|
347
|
+
raise RequestError(f"Cannot read file for upload '{file_path}': {e}") from e
|
|
248
348
|
request_kwargs["files"] = files_list
|
|
249
349
|
|
|
250
350
|
return request_kwargs
|
|
@@ -254,13 +354,13 @@ class Carrier:
|
|
|
254
354
|
try:
|
|
255
355
|
return cls.client.request(**request_kwargs)
|
|
256
356
|
except httpx.TimeoutException as e:
|
|
257
|
-
raise RequestError(f"HTTP request timed out: {e}") from
|
|
357
|
+
raise RequestError(f"HTTP request timed out: {e}") from e
|
|
258
358
|
except httpx.ConnectError as e:
|
|
259
|
-
raise RequestError(f"HTTP connection error: {e}") from
|
|
359
|
+
raise RequestError(f"HTTP connection error: {e}") from e
|
|
260
360
|
except httpx.HTTPError as e:
|
|
261
|
-
raise RequestError(f"HTTP request failed: {e}") from
|
|
361
|
+
raise RequestError(f"HTTP request failed: {e}") from e
|
|
262
362
|
except Exception as e:
|
|
263
|
-
raise RequestError(f"Unexpected error: {e}") from
|
|
363
|
+
raise RequestError(f"Unexpected error: {e}") from e
|
|
264
364
|
|
|
265
365
|
@staticmethod
|
|
266
366
|
def _process_save_step(save_model: Save, response: httpx.Response, context: ChainMap[str, Any]) -> dict[str, Any]:
|
|
@@ -271,21 +371,21 @@ class Carrier:
|
|
|
271
371
|
try:
|
|
272
372
|
response_json = response.json()
|
|
273
373
|
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
274
|
-
raise SaveError(f"Cannot extract variables, response is not valid JSON: {e}") from
|
|
374
|
+
raise SaveError(f"Cannot extract variables, response is not valid JSON: {e}") from e
|
|
275
375
|
|
|
276
376
|
for var_name, jmespath_expr in save_model.jmespath.items():
|
|
277
377
|
try:
|
|
278
378
|
saved_value = jmespath.search(jmespath_expr, response_json)
|
|
279
379
|
step_saved[var_name] = saved_value
|
|
280
380
|
except jmespath.exceptions.JMESPathError as e:
|
|
281
|
-
raise SaveError(f"Error saving variable {var_name}: {e}") from
|
|
381
|
+
raise SaveError(f"Error saving variable {var_name}: {e}") from e
|
|
282
382
|
|
|
283
383
|
case SubstitutionsSave():
|
|
284
384
|
try:
|
|
285
385
|
substitution_result = process_substitutions(save_model.substitutions, context)
|
|
286
386
|
step_saved.update(substitution_result)
|
|
287
387
|
except TemplatesError as e:
|
|
288
|
-
raise SaveError(f"Error processing substitutions: {e}") from
|
|
388
|
+
raise SaveError(f"Error processing substitutions: {e}") from e
|
|
289
389
|
|
|
290
390
|
case UserFunctionsSave():
|
|
291
391
|
for func_item in save_model.user_functions:
|
|
@@ -299,7 +399,7 @@ class Carrier:
|
|
|
299
399
|
except SaveError:
|
|
300
400
|
raise
|
|
301
401
|
except UserFunctionError as e:
|
|
302
|
-
raise SaveError(f"Error calling user function '{func_item}': {e}") from
|
|
402
|
+
raise SaveError(f"Error calling user function '{func_item}': {e}") from e
|
|
303
403
|
|
|
304
404
|
return step_saved
|
|
305
405
|
|
|
@@ -329,7 +429,7 @@ class Carrier:
|
|
|
329
429
|
except VerificationError:
|
|
330
430
|
raise
|
|
331
431
|
except UserFunctionError as e:
|
|
332
|
-
raise VerificationError(f"Error calling user function '{func_item}': {e}") from
|
|
432
|
+
raise VerificationError(f"Error calling user function '{func_item}': {e}") from e
|
|
333
433
|
|
|
334
434
|
if verify_model.body.schema:
|
|
335
435
|
schema = verify_model.body.schema
|
|
@@ -339,21 +439,21 @@ class Carrier:
|
|
|
339
439
|
schema = json.loads(schema_path.read_text())
|
|
340
440
|
check_json_schema(schema)
|
|
341
441
|
except (OSError, json.JSONDecodeError) as e:
|
|
342
|
-
raise VerificationError(f"Error reading body schema file '{schema_path}': {e}") from
|
|
442
|
+
raise VerificationError(f"Error reading body schema file '{schema_path}': {e}") from e
|
|
343
443
|
except jsonschema.SchemaError as e:
|
|
344
|
-
raise VerificationError(f"Invalid JSON Schema in file '{schema_path}': {e}") from
|
|
444
|
+
raise VerificationError(f"Invalid JSON Schema in file '{schema_path}': {e}") from e
|
|
345
445
|
|
|
346
446
|
try:
|
|
347
447
|
response_json = response.json()
|
|
348
448
|
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
349
|
-
raise VerificationError(f"Cannot validate schema, response is not valid JSON: {e}") from
|
|
449
|
+
raise VerificationError(f"Cannot validate schema, response is not valid JSON: {e}") from e
|
|
350
450
|
|
|
351
451
|
try:
|
|
352
452
|
jsonschema.validate(instance=response_json, schema=schema)
|
|
353
453
|
except jsonschema.ValidationError as e:
|
|
354
|
-
raise VerificationError(f"Body schema validation failed: {e}") from
|
|
454
|
+
raise VerificationError(f"Body schema validation failed: {e}") from e
|
|
355
455
|
except jsonschema.SchemaError as e:
|
|
356
|
-
raise VerificationError(f"Invalid body validation schema: {e}") from
|
|
456
|
+
raise VerificationError(f"Invalid body validation schema: {e}") from e
|
|
357
457
|
|
|
358
458
|
for substring in verify_model.body.contains:
|
|
359
459
|
if substring not in response.text:
|
|
@@ -372,16 +472,20 @@ class Carrier:
|
|
|
372
472
|
raise VerificationError(f"Body matches '{pattern}' while it shouldn't")
|
|
373
473
|
|
|
374
474
|
@classmethod
|
|
375
|
-
def _execute_single_iteration(
|
|
475
|
+
def _execute_single_iteration(
|
|
476
|
+
cls, stage: Stage, local_context: ChainMap[str, Any], iter_vars: Mapping[str, Any], limiter: Limiter | None = None, rate_limit_delay: float = 60
|
|
477
|
+
) -> ParallelIterationResult:
|
|
376
478
|
"""Execute a single iteration of a stage."""
|
|
377
479
|
iter_context = local_context.new_child(dict(iter_vars))
|
|
378
480
|
|
|
379
|
-
|
|
380
|
-
|
|
481
|
+
# walk() already returns a re-validated model (it dumps, substitutes, and
|
|
482
|
+
# model_validates when templates are present, else returns the model as-is),
|
|
483
|
+
# so a further model_validate would be a no-op (revalidate_instances='never').
|
|
484
|
+
request_model = walk(stage.request, iter_context)
|
|
381
485
|
request_kwargs = cls._build_request_kwargs(request_model)
|
|
382
486
|
|
|
383
|
-
if limiter:
|
|
384
|
-
|
|
487
|
+
if limiter is not None and not limiter.try_acquire("api", blocking=True, timeout=rate_limit_delay):
|
|
488
|
+
raise RequestError(f"Rate limit exceeded: could not acquire a request slot within {rate_limit_delay}s")
|
|
385
489
|
|
|
386
490
|
response = cls._execute_http_request(request_kwargs)
|
|
387
491
|
|
|
@@ -390,15 +494,13 @@ class Carrier:
|
|
|
390
494
|
for step in stage.response:
|
|
391
495
|
match step:
|
|
392
496
|
case SaveStep():
|
|
393
|
-
|
|
394
|
-
save_model = TypeAdapter(Save).validate_python(save_dict)
|
|
497
|
+
save_model = walk(step.save, iter_context)
|
|
395
498
|
step_saved = cls._process_save_step(save_model, response, iter_context)
|
|
396
499
|
iter_context = iter_context.new_child(step_saved)
|
|
397
500
|
saved_context.update(step_saved)
|
|
398
501
|
|
|
399
502
|
case VerifyStep():
|
|
400
|
-
|
|
401
|
-
verify_model = Verify.model_validate(verify_dict)
|
|
503
|
+
verify_model = walk(step.verify, iter_context)
|
|
402
504
|
cls._process_verify_step(verify_model, response)
|
|
403
505
|
except StageExecutionError as e:
|
|
404
506
|
e.request = response.request
|
|
@@ -519,7 +621,7 @@ def create_test_class(scenario: Scenario, class_name: str, max_parallel_iteratio
|
|
|
519
621
|
parametrize_marker = pytest.mark.parametrize(",".join(param_names), param_values, ids=param_ids)
|
|
520
622
|
stage_method = parametrize_marker(stage_method)
|
|
521
623
|
|
|
522
|
-
all_fixtures = ["self"] + all_param_names + stage.fixtures
|
|
624
|
+
all_fixtures = ["self"] + list(dict.fromkeys(all_param_names + stage.fixtures + scenario.fixtures))
|
|
523
625
|
stage_method.__signature__ = inspect.Signature([inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD) for name in all_fixtures]) # type: ignore[assignment]
|
|
524
626
|
|
|
525
627
|
all_marks = [f"order({i})"] + stage.marks
|
|
@@ -527,7 +629,11 @@ def create_test_class(scenario: Scenario, class_name: str, max_parallel_iteratio
|
|
|
527
629
|
try:
|
|
528
630
|
stage_method = make_marker(mark_str)(stage_method)
|
|
529
631
|
except Exception as e:
|
|
530
|
-
|
|
632
|
+
# A malformed stage marker is an author error: fail collection (the
|
|
633
|
+
# caller wraps this into a CollectError) instead of silently dropping
|
|
634
|
+
# the marker and running the stage — matching how scenario-level
|
|
635
|
+
# markers are handled in plugin.py.
|
|
636
|
+
raise StageExecutionError(f"Invalid marker '{mark_str}' on stage '{stage.name}': {e}") from e
|
|
531
637
|
|
|
532
638
|
method_name = f"test {str(i).zfill(padding_width)} - {stage.name}"
|
|
533
639
|
setattr(CustomCarrier, method_name, stage_method)
|