pytest-httpchain 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/PKG-INFO +10 -4
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/README.md +1 -1
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/pyproject.toml +9 -14
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/__init__.py +7 -0
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/carrier.py +206 -115
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/cli.py +60 -36
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/constants.py +3 -2
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/dataflow.py +8 -24
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/har_writer.py +23 -4
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/plugin.py +75 -36
- pytest_httpchain-0.7.0/src/pytest_httpchain/report_formatter.py +88 -0
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/schema.py +13 -7
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/utils.py +38 -0
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/validation.py +170 -93
- pytest_httpchain-0.6.0/src/pytest_httpchain/report_formatter.py +0 -69
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/LICENSE +0 -0
- {pytest_httpchain-0.6.0 → pytest_httpchain-0.7.0}/src/pytest_httpchain/exceptions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pytest-httpchain
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: pytest plugin for HTTP testing using JSON files
|
|
5
5
|
Keywords: testing,pytest,requests
|
|
6
6
|
Author: Alexander Eresov
|
|
@@ -15,14 +15,20 @@ Classifier: Programming Language :: Python :: 3
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
16
|
Classifier: Topic :: Software Development :: Testing
|
|
17
17
|
Requires-Dist: httpx[http2]>=0.27.0
|
|
18
|
+
Requires-Dist: jmespath>=1.0.0
|
|
19
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
18
20
|
Requires-Dist: pydantic>=2.11.7
|
|
19
|
-
Requires-Dist: pyrate-limiter>=
|
|
21
|
+
Requires-Dist: pyrate-limiter>=4.0.0
|
|
22
|
+
Requires-Dist: pytest>=8.4
|
|
23
|
+
Requires-Dist: pytest-httpchain-core
|
|
20
24
|
Requires-Dist: pytest-httpchain-jsonref
|
|
21
25
|
Requires-Dist: pytest-httpchain-models
|
|
22
26
|
Requires-Dist: pytest-httpchain-templates
|
|
27
|
+
Requires-Dist: pytest-httpchain-userfunc
|
|
23
28
|
Requires-Dist: pytest-order>=1.3.0
|
|
29
|
+
Requires-Dist: simpleeval>=1.0.3
|
|
24
30
|
Requires-Dist: typer>=0.16.0
|
|
25
|
-
Requires-Python: >=3.13
|
|
31
|
+
Requires-Python: >=3.13
|
|
26
32
|
Description-Content-Type: text/markdown
|
|
27
33
|
|
|
28
34
|
[](https://pypi.python.org/pypi/pytest-httpchain)
|
|
@@ -71,7 +77,7 @@ A key-value store persists throughout scenario execution. Variables, fixtures, a
|
|
|
71
77
|
|
|
72
78
|
- **JMESPath** — Extract values from JSON responses directly
|
|
73
79
|
- **JSON Schema** — Validate response structure against a schema
|
|
74
|
-
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://
|
|
80
|
+
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://www.python-httpx.org/advanced/authentication/#custom-authentication-schemes)
|
|
75
81
|
|
|
76
82
|
### Full pytest integration
|
|
77
83
|
|
|
@@ -44,7 +44,7 @@ A key-value store persists throughout scenario execution. Variables, fixtures, a
|
|
|
44
44
|
|
|
45
45
|
- **JMESPath** — Extract values from JSON responses directly
|
|
46
46
|
- **JSON Schema** — Validate response structure against a schema
|
|
47
|
-
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://
|
|
47
|
+
- **User functions** — Call Python functions for custom extraction, verification, or [authentication](https://www.python-httpx.org/advanced/authentication/#custom-authentication-schemes)
|
|
48
48
|
|
|
49
49
|
### Full pytest integration
|
|
50
50
|
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pytest-httpchain"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.0"
|
|
4
4
|
description = "pytest plugin for HTTP testing using JSON files"
|
|
5
5
|
readme = "README.md"
|
|
6
|
-
requires-python = ">=3.13
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
7
|
authors = [{ name = "Alexander Eresov", email = "aeresov@gmail.com" }]
|
|
8
8
|
dependencies = [
|
|
9
9
|
"httpx[http2]>=0.27.0",
|
|
10
|
+
"jmespath>=1.0.0",
|
|
11
|
+
"jsonschema>=4.0.0",
|
|
10
12
|
"pydantic>=2.11.7",
|
|
11
|
-
"pyrate-limiter>=
|
|
13
|
+
"pyrate-limiter>=4.0.0",
|
|
14
|
+
"pytest>=8.4",
|
|
15
|
+
"pytest-httpchain-core",
|
|
12
16
|
"pytest-httpchain-jsonref",
|
|
13
17
|
"pytest-httpchain-models",
|
|
14
18
|
"pytest-httpchain-templates",
|
|
19
|
+
"pytest-httpchain-userfunc",
|
|
15
20
|
"pytest-order>=1.3.0",
|
|
21
|
+
"simpleeval>=1.0.3",
|
|
16
22
|
"typer>=0.16.0",
|
|
17
23
|
]
|
|
18
24
|
keywords = ["testing", "pytest", "requests"]
|
|
@@ -40,8 +46,6 @@ dev = [
|
|
|
40
46
|
docs = [
|
|
41
47
|
"mkdocs>=1.6.1",
|
|
42
48
|
"mkdocs-material>=9.6.16",
|
|
43
|
-
"mkdocstrings>=0.30.0",
|
|
44
|
-
"mkdocstrings-python>=1.16.12",
|
|
45
49
|
]
|
|
46
50
|
|
|
47
51
|
[tool.uv]
|
|
@@ -107,15 +111,6 @@ pythonpath = [
|
|
|
107
111
|
]
|
|
108
112
|
testpaths = ["tests", "packages/*/tests"]
|
|
109
113
|
|
|
110
|
-
[tool.pyright]
|
|
111
|
-
exclude = [".tox/"]
|
|
112
|
-
ignore = [".venv/"]
|
|
113
|
-
reportMissingImports = "error"
|
|
114
|
-
reportMissingTypeStubs = false
|
|
115
|
-
reportExplicitAny = false
|
|
116
|
-
typeCheckingMode = "off"
|
|
117
|
-
pythonPlatform = "Linux"
|
|
118
|
-
|
|
119
114
|
[tool.coverage.run]
|
|
120
115
|
source = [
|
|
121
116
|
"src",
|
|
@@ -21,3 +21,10 @@ Example test file (test_api.http.json):
|
|
|
21
21
|
]
|
|
22
22
|
}
|
|
23
23
|
"""
|
|
24
|
+
|
|
25
|
+
# Re-export the user-facing warning so consumers can filter it without reaching
|
|
26
|
+
# into the plugin module. The definition stays in plugin.py because the pytest11
|
|
27
|
+
# entry point and the collection hook reference it there directly.
|
|
28
|
+
from pytest_httpchain.plugin import ScenarioValidationWarning
|
|
29
|
+
|
|
30
|
+
__all__ = ["ScenarioValidationWarning"]
|
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
"""Test execution engine for pytest-httpchain scenarios.
|
|
2
|
+
|
|
3
|
+
``create_test_class`` turns a validated :class:`Scenario` into a dynamic pytest
|
|
4
|
+
test class (a subclass of :class:`Carrier`), one ``test_NN - <stage name>``
|
|
5
|
+
method per stage, ordered by the ``order(i)`` marker so the stages run as a
|
|
6
|
+
chain. Each scenario gets its own subclass; the per-scenario mutable state
|
|
7
|
+
(``client``, ``global_context``, ``aborted``, ``last_request``/``last_response``,
|
|
8
|
+
``active_context_managers``) lives at the *class* level and is overridden in the
|
|
9
|
+
subclass dict, so the stage methods — which are classmethods operating on
|
|
10
|
+
``cls`` — share one running context across the chain while different scenarios
|
|
11
|
+
stay isolated from each other.
|
|
12
|
+
|
|
13
|
+
Per stage, :meth:`Carrier.execute_stage` gates on the abort/``always_run`` flow,
|
|
14
|
+
layers fixtures and substitutions over the global context, expands any
|
|
15
|
+
``parallel`` config into iterations (sequential or thread-pooled, optionally
|
|
16
|
+
rate limited), executes the HTTP request via httpx, runs the response
|
|
17
|
+
verify/save steps, and on full success commits the collected saves as a new
|
|
18
|
+
global-context layer for later stages. Expected failures (bad scenario, failed
|
|
19
|
+
verification, unreachable server) are surfaced via ``pytest.fail(pytrace=False)``
|
|
20
|
+
so the report stays clean rather than dumping an internal traceback.
|
|
21
|
+
"""
|
|
22
|
+
|
|
1
23
|
import base64
|
|
2
24
|
import inspect
|
|
3
25
|
import json
|
|
@@ -17,7 +39,7 @@ import jmespath
|
|
|
17
39
|
import jmespath.exceptions
|
|
18
40
|
import jsonschema
|
|
19
41
|
import pytest
|
|
20
|
-
from pydantic import
|
|
42
|
+
from pydantic import ValidationError
|
|
21
43
|
from pyrate_limiter import Duration, Limiter, Rate
|
|
22
44
|
from pytest_httpchain_models import (
|
|
23
45
|
Base64Body,
|
|
@@ -53,6 +75,12 @@ from .utils import call_user_function, make_marker, process_substitutions
|
|
|
53
75
|
|
|
54
76
|
logger = logging.getLogger(__name__)
|
|
55
77
|
|
|
78
|
+
# Exceptions that represent an expected stage failure (a bad scenario, a failed
|
|
79
|
+
# verification, an unreachable server) rather than a bug in the plugin. Both the
|
|
80
|
+
# sequential and the parallel execution paths catch these and turn them into a
|
|
81
|
+
# clean pytest failure instead of a raw traceback.
|
|
82
|
+
_STAGE_FAILURE_EXCEPTIONS = (StageExecutionError, TemplatesError, ValidationError)
|
|
83
|
+
|
|
56
84
|
|
|
57
85
|
@dataclass
|
|
58
86
|
class ParallelIterationResult:
|
|
@@ -94,15 +122,22 @@ class Carrier:
|
|
|
94
122
|
|
|
95
123
|
@classmethod
|
|
96
124
|
def execute_stage(cls, stage: Stage, fixture_kwargs: dict[str, Any]) -> None:
|
|
125
|
+
"""Execute one stage end to end.
|
|
126
|
+
|
|
127
|
+
Gates on the abort/``always_run`` flow, layers the stage context
|
|
128
|
+
(fixtures + substitutions over the global context), builds the iteration
|
|
129
|
+
matrix (``_build_iteration_substitutions``), runs the iterations
|
|
130
|
+
sequentially or across a thread pool (``_run_iterations``), and on full
|
|
131
|
+
success commits the collected saves as a new global-context layer.
|
|
132
|
+
|
|
133
|
+
Any expected failure is reported via ``pytest.fail`` (clean, no
|
|
134
|
+
traceback) and marks the chain aborted so later stages skip unless they
|
|
135
|
+
opt into ``always_run``. A failing stage commits **no** saves, so the
|
|
136
|
+
global context is left unchanged (deterministic) rather than carrying a
|
|
137
|
+
thread-timing-dependent subset of a parallel run.
|
|
138
|
+
"""
|
|
97
139
|
try:
|
|
98
|
-
|
|
99
|
-
# template sees the same fixture layer as stage templates)
|
|
100
|
-
stage_fixtures: dict[str, Any] = {}
|
|
101
|
-
for name, value in fixture_kwargs.items():
|
|
102
|
-
if callable(value) and not inspect.isclass(value):
|
|
103
|
-
stage_fixtures[name] = cls._wrap_factory_fixture(value)
|
|
104
|
-
else:
|
|
105
|
-
stage_fixtures[name] = value
|
|
140
|
+
stage_fixtures = cls._build_stage_fixtures(fixture_kwargs)
|
|
106
141
|
|
|
107
142
|
if cls.aborted and not cls._resolve_always_run(stage, stage_fixtures):
|
|
108
143
|
pytest.skip(reason="Flow aborted")
|
|
@@ -115,99 +150,145 @@ class Carrier:
|
|
|
115
150
|
logger.info(f"global context on start: {json.dumps(dict(cls.global_context), indent=2, default=str)}")
|
|
116
151
|
logger.info(f"local context on start: {json.dumps(dict(local_context), indent=2, default=str)}")
|
|
117
152
|
|
|
118
|
-
# build iterations
|
|
119
|
-
iteration_substitutions: list[dict[str, Any]] = [{}]
|
|
120
153
|
parallel_config = walk(stage.parallel, local_context) if stage.parallel else None
|
|
121
|
-
|
|
122
|
-
case None:
|
|
123
|
-
pass
|
|
124
|
-
case ParallelRepeatConfig(repeat=repeat_count):
|
|
125
|
-
iteration_substitutions = [{} for _ in range(repeat_count)]
|
|
126
|
-
case ParallelForeachConfig(foreach=foreach_steps):
|
|
127
|
-
# same algorithm like what pytest does for parametrize marker
|
|
128
|
-
for step in foreach_steps:
|
|
129
|
-
match step:
|
|
130
|
-
case IndividualParameter(individual=individual):
|
|
131
|
-
param_name = next(iter(individual.keys()))
|
|
132
|
-
values = individual[param_name]
|
|
133
|
-
iteration_substitutions = [{**existing, param_name: val} for val in values for existing in iteration_substitutions]
|
|
134
|
-
case CombinationsParameter(combinations=combinations):
|
|
135
|
-
combos: list[dict[str, Any] | SimpleNamespace] = [vars(item) if isinstance(item, SimpleNamespace) else item for item in combinations]
|
|
136
|
-
iteration_substitutions = [{**existing, **combo} for combo in combos for existing in iteration_substitutions]
|
|
137
|
-
|
|
138
|
-
if len(iteration_substitutions) > cls.max_parallel_iterations:
|
|
139
|
-
raise StageExecutionError(
|
|
140
|
-
f"Parallel iteration count ({len(iteration_substitutions)}) exceeds maximum ({cls.max_parallel_iterations}). "
|
|
141
|
-
f"Set 'max_parallel_iterations' in pytest.ini to increase the limit."
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
# execute iterations
|
|
145
|
-
max_concurrency = parallel_config.max_concurrency if parallel_config else 1
|
|
146
|
-
calls_per_sec = parallel_config.calls_per_sec if parallel_config else None
|
|
147
|
-
max_rate_limit_delay = parallel_config.max_rate_limit_delay if parallel_config else 60
|
|
154
|
+
iteration_substitutions = cls._build_iteration_substitutions(parallel_config)
|
|
148
155
|
|
|
149
156
|
total = len(iteration_substitutions)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
157
|
+
if total == 0:
|
|
158
|
+
# Unreachable via validated models (foreach values have min_length=1,
|
|
159
|
+
# repeat is PositiveInt), but guard so a future gap fails loudly
|
|
160
|
+
# instead of passing a stage that never sent a request.
|
|
161
|
+
raise StageExecutionError("Parallel configuration produced zero iterations; foreach/repeat must yield at least one item")
|
|
162
|
+
if total > cls.max_parallel_iterations:
|
|
163
|
+
raise StageExecutionError(
|
|
164
|
+
f"Parallel iteration count ({total}) exceeds maximum ({cls.max_parallel_iterations}). Set 'max_parallel_iterations' in pytest.ini to increase the limit."
|
|
165
|
+
)
|
|
153
166
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
results, first_error = cls._run_iterations(stage, local_context, iteration_substitutions, parallel_config)
|
|
168
|
+
|
|
169
|
+
if first_error is None:
|
|
170
|
+
# Commit saves only on full success. A failed (parallel) stage must
|
|
171
|
+
# leave the global context untouched, not commit a non-deterministic
|
|
172
|
+
# subset of iterations whose saves happened to land before the error.
|
|
173
|
+
all_saves: dict[str, Any] = {}
|
|
174
|
+
for iter_result in results:
|
|
175
|
+
if iter_result is not None:
|
|
176
|
+
all_saves.update(iter_result.saved_context)
|
|
177
|
+
cls.last_request = iter_result.request
|
|
178
|
+
cls.last_response = iter_result.response
|
|
179
|
+
logger.info(f"updates for global context: {json.dumps(all_saves, indent=2, default=str)}")
|
|
180
|
+
cls.global_context = cls.global_context.new_child(all_saves)
|
|
159
181
|
else:
|
|
160
|
-
workers = min(max_concurrency, total) if total > 0 else 1
|
|
161
|
-
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
162
|
-
futures: dict[Future[ParallelIterationResult], int] = {}
|
|
163
|
-
for idx, iter_vars in enumerate(iteration_substitutions):
|
|
164
|
-
future = executor.submit(cls._execute_single_iteration, stage, local_context, iter_vars, limiter)
|
|
165
|
-
futures[future] = idx
|
|
166
|
-
|
|
167
|
-
for future in as_completed(futures):
|
|
168
|
-
idx = futures[future]
|
|
169
|
-
try:
|
|
170
|
-
results[idx] = future.result()
|
|
171
|
-
except (StageExecutionError, TemplatesError, ValidationError) as e:
|
|
172
|
-
first_error = (idx, e)
|
|
173
|
-
executor.shutdown(wait=False, cancel_futures=True)
|
|
174
|
-
break
|
|
175
|
-
|
|
176
|
-
# Apply results in index order - collect all saves for new layer
|
|
177
|
-
all_saves: dict[str, Any] = {}
|
|
178
|
-
for iter_result in results:
|
|
179
|
-
if iter_result is not None:
|
|
180
|
-
all_saves.update(iter_result.saved_context)
|
|
181
|
-
cls.last_request = iter_result.request
|
|
182
|
-
cls.last_response = iter_result.response
|
|
183
|
-
|
|
184
|
-
logger.info(f"updates for global context: {json.dumps(all_saves, indent=2, default=str)}")
|
|
185
|
-
|
|
186
|
-
# Add response saves as new layer
|
|
187
|
-
cls.global_context = cls.global_context.new_child(all_saves)
|
|
188
|
-
|
|
189
|
-
# Handle error
|
|
190
|
-
if first_error:
|
|
191
182
|
idx, exc = first_error
|
|
192
|
-
#
|
|
183
|
+
# Surface the failed iteration's request/response in the report.
|
|
193
184
|
if isinstance(exc, StageExecutionError):
|
|
194
185
|
if exc.request is not None:
|
|
195
186
|
cls.last_request = exc.request
|
|
196
187
|
if exc.response is not None:
|
|
197
188
|
cls.last_response = exc.response
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
189
|
+
# Only label the failure as parallel when the user configured
|
|
190
|
+
# `parallel`; otherwise re-raise the original error unchanged so a
|
|
191
|
+
# plain stage failure isn't misreported as "Parallel execution failed".
|
|
192
|
+
if parallel_config is not None:
|
|
193
|
+
raise StageExecutionError(f"Parallel execution failed at iteration {idx}: {exc}") from exc
|
|
194
|
+
raise exc
|
|
195
|
+
|
|
196
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
197
|
+
# Detect xfail structurally (marker name == "xfail") rather than by a
|
|
198
|
+
# substring scan of the raw mark string, so e.g. `skip(reason="...xfail...")`
|
|
199
|
+
# or a custom `my_xfail` marker is not misclassified. Every mark already
|
|
200
|
+
# round-tripped through make_marker() at collection time (create_test_class),
|
|
201
|
+
# so parsing here cannot raise on a previously-validated scenario.
|
|
202
|
+
is_xfail = any(make_marker(mark).name == "xfail" for mark in stage.marks)
|
|
206
203
|
if not is_xfail:
|
|
207
204
|
logger.error(str(e))
|
|
208
205
|
cls.aborted = True
|
|
209
206
|
pytest.fail(reason=str(e), pytrace=False)
|
|
210
207
|
|
|
208
|
+
@classmethod
|
|
209
|
+
def _build_stage_fixtures(cls, fixture_kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
210
|
+
"""Resolve injected fixtures, wrapping callable (factory) fixtures so they
|
|
211
|
+
can be invoked from template expressions while plain values pass through."""
|
|
212
|
+
stage_fixtures: dict[str, Any] = {}
|
|
213
|
+
for name, value in fixture_kwargs.items():
|
|
214
|
+
if callable(value) and not inspect.isclass(value):
|
|
215
|
+
stage_fixtures[name] = cls._wrap_factory_fixture(value)
|
|
216
|
+
else:
|
|
217
|
+
stage_fixtures[name] = value
|
|
218
|
+
return stage_fixtures
|
|
219
|
+
|
|
220
|
+
@staticmethod
|
|
221
|
+
def _build_iteration_substitutions(parallel_config: Any) -> list[dict[str, Any]]:
|
|
222
|
+
"""Expand the (already template-resolved) parallel config into per-iteration
|
|
223
|
+
substitution dicts: non-parallel -> a single empty dict; ``repeat`` -> N
|
|
224
|
+
empties; ``foreach`` -> the cross-product of its parameter steps (the same
|
|
225
|
+
algorithm pytest uses for the ``parametrize`` marker)."""
|
|
226
|
+
iteration_substitutions: list[dict[str, Any]] = [{}]
|
|
227
|
+
match parallel_config:
|
|
228
|
+
case None:
|
|
229
|
+
pass
|
|
230
|
+
case ParallelRepeatConfig(repeat=repeat_count):
|
|
231
|
+
iteration_substitutions = [{} for _ in range(repeat_count)]
|
|
232
|
+
case ParallelForeachConfig(foreach=foreach_steps):
|
|
233
|
+
for step in foreach_steps:
|
|
234
|
+
match step:
|
|
235
|
+
case IndividualParameter(individual=individual):
|
|
236
|
+
param_name = next(iter(individual.keys()))
|
|
237
|
+
values = individual[param_name]
|
|
238
|
+
iteration_substitutions = [{**existing, param_name: val} for val in values for existing in iteration_substitutions]
|
|
239
|
+
case CombinationsParameter(combinations=combinations):
|
|
240
|
+
combos: list[dict[str, Any] | SimpleNamespace] = [vars(item) if isinstance(item, SimpleNamespace) else item for item in combinations]
|
|
241
|
+
iteration_substitutions = [{**existing, **combo} for combo in combos for existing in iteration_substitutions]
|
|
242
|
+
return iteration_substitutions
|
|
243
|
+
|
|
244
|
+
@classmethod
|
|
245
|
+
def _run_iterations(
|
|
246
|
+
cls,
|
|
247
|
+
stage: Stage,
|
|
248
|
+
local_context: ChainMap[str, Any],
|
|
249
|
+
iteration_substitutions: list[dict[str, Any]],
|
|
250
|
+
parallel_config: Any,
|
|
251
|
+
) -> tuple[list[ParallelIterationResult | None], tuple[int, Exception] | None]:
|
|
252
|
+
"""Run the iterations and return ``(results_by_index, first_error)``.
|
|
253
|
+
|
|
254
|
+
A single iteration runs inline; multiple iterations run in a
|
|
255
|
+
``ThreadPoolExecutor`` capped at ``max_concurrency``, with an optional
|
|
256
|
+
global rate limiter. On the first expected failure the pool is cancelled
|
|
257
|
+
and ``(index, exception)`` is returned; otherwise ``first_error`` is None.
|
|
258
|
+
"""
|
|
259
|
+
max_concurrency = parallel_config.max_concurrency if parallel_config else 1
|
|
260
|
+
calls_per_sec = parallel_config.calls_per_sec if parallel_config else None
|
|
261
|
+
max_rate_limit_delay = parallel_config.max_rate_limit_delay if parallel_config else 60
|
|
262
|
+
|
|
263
|
+
total = len(iteration_substitutions)
|
|
264
|
+
results: list[ParallelIterationResult | None] = [None] * total
|
|
265
|
+
first_error: tuple[int, Exception] | None = None
|
|
266
|
+
limiter = Limiter(Rate(calls_per_sec, Duration.SECOND)) if calls_per_sec else None
|
|
267
|
+
|
|
268
|
+
if total == 1:
|
|
269
|
+
try:
|
|
270
|
+
results[0] = cls._execute_single_iteration(stage, local_context, iteration_substitutions[0], limiter, max_rate_limit_delay)
|
|
271
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
272
|
+
first_error = (0, e)
|
|
273
|
+
else:
|
|
274
|
+
workers = min(max_concurrency, total)
|
|
275
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
276
|
+
futures: dict[Future[ParallelIterationResult], int] = {}
|
|
277
|
+
for idx, iter_vars in enumerate(iteration_substitutions):
|
|
278
|
+
future = executor.submit(cls._execute_single_iteration, stage, local_context, iter_vars, limiter, max_rate_limit_delay)
|
|
279
|
+
futures[future] = idx
|
|
280
|
+
|
|
281
|
+
for future in as_completed(futures):
|
|
282
|
+
idx = futures[future]
|
|
283
|
+
try:
|
|
284
|
+
results[idx] = future.result()
|
|
285
|
+
except _STAGE_FAILURE_EXCEPTIONS as e:
|
|
286
|
+
first_error = (idx, e)
|
|
287
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
288
|
+
break
|
|
289
|
+
|
|
290
|
+
return results, first_error
|
|
291
|
+
|
|
211
292
|
@staticmethod
|
|
212
293
|
def _build_request_kwargs(request_model: Request) -> dict[str, Any]:
|
|
213
294
|
request_kwargs: dict[str, Any] = {
|
|
@@ -224,7 +305,7 @@ class Carrier:
|
|
|
224
305
|
auth_result = call_user_function(request_model.auth)
|
|
225
306
|
request_kwargs["auth"] = auth_result
|
|
226
307
|
except UserFunctionError as e:
|
|
227
|
-
raise RequestError(f"Failed to configure authentication: {e}") from
|
|
308
|
+
raise RequestError(f"Failed to configure authentication: {e}") from e
|
|
228
309
|
|
|
229
310
|
match request_model.body:
|
|
230
311
|
case None:
|
|
@@ -249,8 +330,10 @@ class Carrier:
|
|
|
249
330
|
case BinaryBody(binary=file_path):
|
|
250
331
|
try:
|
|
251
332
|
request_kwargs["content"] = Path(file_path).read_bytes()
|
|
252
|
-
except FileNotFoundError:
|
|
253
|
-
raise RequestError(f"Binary file not found: {file_path}") from
|
|
333
|
+
except FileNotFoundError as e:
|
|
334
|
+
raise RequestError(f"Binary file not found: {file_path}") from e
|
|
335
|
+
except OSError as e:
|
|
336
|
+
raise RequestError(f"Cannot read binary file '{file_path}': {e}") from e
|
|
254
337
|
|
|
255
338
|
case FilesBody(files=file_paths):
|
|
256
339
|
files_list = []
|
|
@@ -258,8 +341,10 @@ class Carrier:
|
|
|
258
341
|
path = Path(file_path)
|
|
259
342
|
try:
|
|
260
343
|
files_list.append((field_name, (path.name, path.read_bytes())))
|
|
261
|
-
except FileNotFoundError:
|
|
262
|
-
raise RequestError(f"File not found for upload: {file_path}") from
|
|
344
|
+
except FileNotFoundError as e:
|
|
345
|
+
raise RequestError(f"File not found for upload: {file_path}") from e
|
|
346
|
+
except OSError as e:
|
|
347
|
+
raise RequestError(f"Cannot read file for upload '{file_path}': {e}") from e
|
|
263
348
|
request_kwargs["files"] = files_list
|
|
264
349
|
|
|
265
350
|
return request_kwargs
|
|
@@ -269,13 +354,13 @@ class Carrier:
|
|
|
269
354
|
try:
|
|
270
355
|
return cls.client.request(**request_kwargs)
|
|
271
356
|
except httpx.TimeoutException as e:
|
|
272
|
-
raise RequestError(f"HTTP request timed out: {e}") from
|
|
357
|
+
raise RequestError(f"HTTP request timed out: {e}") from e
|
|
273
358
|
except httpx.ConnectError as e:
|
|
274
|
-
raise RequestError(f"HTTP connection error: {e}") from
|
|
359
|
+
raise RequestError(f"HTTP connection error: {e}") from e
|
|
275
360
|
except httpx.HTTPError as e:
|
|
276
|
-
raise RequestError(f"HTTP request failed: {e}") from
|
|
361
|
+
raise RequestError(f"HTTP request failed: {e}") from e
|
|
277
362
|
except Exception as e:
|
|
278
|
-
raise RequestError(f"Unexpected error: {e}") from
|
|
363
|
+
raise RequestError(f"Unexpected error: {e}") from e
|
|
279
364
|
|
|
280
365
|
@staticmethod
|
|
281
366
|
def _process_save_step(save_model: Save, response: httpx.Response, context: ChainMap[str, Any]) -> dict[str, Any]:
|
|
@@ -286,21 +371,21 @@ class Carrier:
|
|
|
286
371
|
try:
|
|
287
372
|
response_json = response.json()
|
|
288
373
|
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
289
|
-
raise SaveError(f"Cannot extract variables, response is not valid JSON: {e}") from
|
|
374
|
+
raise SaveError(f"Cannot extract variables, response is not valid JSON: {e}") from e
|
|
290
375
|
|
|
291
376
|
for var_name, jmespath_expr in save_model.jmespath.items():
|
|
292
377
|
try:
|
|
293
378
|
saved_value = jmespath.search(jmespath_expr, response_json)
|
|
294
379
|
step_saved[var_name] = saved_value
|
|
295
380
|
except jmespath.exceptions.JMESPathError as e:
|
|
296
|
-
raise SaveError(f"Error saving variable {var_name}: {e}") from
|
|
381
|
+
raise SaveError(f"Error saving variable {var_name}: {e}") from e
|
|
297
382
|
|
|
298
383
|
case SubstitutionsSave():
|
|
299
384
|
try:
|
|
300
385
|
substitution_result = process_substitutions(save_model.substitutions, context)
|
|
301
386
|
step_saved.update(substitution_result)
|
|
302
387
|
except TemplatesError as e:
|
|
303
|
-
raise SaveError(f"Error processing substitutions: {e}") from
|
|
388
|
+
raise SaveError(f"Error processing substitutions: {e}") from e
|
|
304
389
|
|
|
305
390
|
case UserFunctionsSave():
|
|
306
391
|
for func_item in save_model.user_functions:
|
|
@@ -314,7 +399,7 @@ class Carrier:
|
|
|
314
399
|
except SaveError:
|
|
315
400
|
raise
|
|
316
401
|
except UserFunctionError as e:
|
|
317
|
-
raise SaveError(f"Error calling user function '{func_item}': {e}") from
|
|
402
|
+
raise SaveError(f"Error calling user function '{func_item}': {e}") from e
|
|
318
403
|
|
|
319
404
|
return step_saved
|
|
320
405
|
|
|
@@ -344,7 +429,7 @@ class Carrier:
|
|
|
344
429
|
except VerificationError:
|
|
345
430
|
raise
|
|
346
431
|
except UserFunctionError as e:
|
|
347
|
-
raise VerificationError(f"Error calling user function '{func_item}': {e}") from
|
|
432
|
+
raise VerificationError(f"Error calling user function '{func_item}': {e}") from e
|
|
348
433
|
|
|
349
434
|
if verify_model.body.schema:
|
|
350
435
|
schema = verify_model.body.schema
|
|
@@ -354,21 +439,21 @@ class Carrier:
|
|
|
354
439
|
schema = json.loads(schema_path.read_text())
|
|
355
440
|
check_json_schema(schema)
|
|
356
441
|
except (OSError, json.JSONDecodeError) as e:
|
|
357
|
-
raise VerificationError(f"Error reading body schema file '{schema_path}': {e}") from
|
|
442
|
+
raise VerificationError(f"Error reading body schema file '{schema_path}': {e}") from e
|
|
358
443
|
except jsonschema.SchemaError as e:
|
|
359
|
-
raise VerificationError(f"Invalid JSON Schema in file '{schema_path}': {e}") from
|
|
444
|
+
raise VerificationError(f"Invalid JSON Schema in file '{schema_path}': {e}") from e
|
|
360
445
|
|
|
361
446
|
try:
|
|
362
447
|
response_json = response.json()
|
|
363
448
|
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
364
|
-
raise VerificationError(f"Cannot validate schema, response is not valid JSON: {e}") from
|
|
449
|
+
raise VerificationError(f"Cannot validate schema, response is not valid JSON: {e}") from e
|
|
365
450
|
|
|
366
451
|
try:
|
|
367
452
|
jsonschema.validate(instance=response_json, schema=schema)
|
|
368
453
|
except jsonschema.ValidationError as e:
|
|
369
|
-
raise VerificationError(f"Body schema validation failed: {e}") from
|
|
454
|
+
raise VerificationError(f"Body schema validation failed: {e}") from e
|
|
370
455
|
except jsonschema.SchemaError as e:
|
|
371
|
-
raise VerificationError(f"Invalid body validation schema: {e}") from
|
|
456
|
+
raise VerificationError(f"Invalid body validation schema: {e}") from e
|
|
372
457
|
|
|
373
458
|
for substring in verify_model.body.contains:
|
|
374
459
|
if substring not in response.text:
|
|
@@ -387,16 +472,20 @@ class Carrier:
|
|
|
387
472
|
raise VerificationError(f"Body matches '{pattern}' while it shouldn't")
|
|
388
473
|
|
|
389
474
|
@classmethod
|
|
390
|
-
def _execute_single_iteration(
|
|
475
|
+
def _execute_single_iteration(
|
|
476
|
+
cls, stage: Stage, local_context: ChainMap[str, Any], iter_vars: Mapping[str, Any], limiter: Limiter | None = None, rate_limit_delay: float = 60
|
|
477
|
+
) -> ParallelIterationResult:
|
|
391
478
|
"""Execute a single iteration of a stage."""
|
|
392
479
|
iter_context = local_context.new_child(dict(iter_vars))
|
|
393
480
|
|
|
394
|
-
|
|
395
|
-
|
|
481
|
+
# walk() already returns a re-validated model (it dumps, substitutes, and
|
|
482
|
+
# model_validates when templates are present, else returns the model as-is),
|
|
483
|
+
# so a further model_validate would be a no-op (revalidate_instances='never').
|
|
484
|
+
request_model = walk(stage.request, iter_context)
|
|
396
485
|
request_kwargs = cls._build_request_kwargs(request_model)
|
|
397
486
|
|
|
398
|
-
if limiter:
|
|
399
|
-
|
|
487
|
+
if limiter is not None and not limiter.try_acquire("api", blocking=True, timeout=rate_limit_delay):
|
|
488
|
+
raise RequestError(f"Rate limit exceeded: could not acquire a request slot within {rate_limit_delay}s")
|
|
400
489
|
|
|
401
490
|
response = cls._execute_http_request(request_kwargs)
|
|
402
491
|
|
|
@@ -405,15 +494,13 @@ class Carrier:
|
|
|
405
494
|
for step in stage.response:
|
|
406
495
|
match step:
|
|
407
496
|
case SaveStep():
|
|
408
|
-
|
|
409
|
-
save_model = TypeAdapter(Save).validate_python(save_dict)
|
|
497
|
+
save_model = walk(step.save, iter_context)
|
|
410
498
|
step_saved = cls._process_save_step(save_model, response, iter_context)
|
|
411
499
|
iter_context = iter_context.new_child(step_saved)
|
|
412
500
|
saved_context.update(step_saved)
|
|
413
501
|
|
|
414
502
|
case VerifyStep():
|
|
415
|
-
|
|
416
|
-
verify_model = Verify.model_validate(verify_dict)
|
|
503
|
+
verify_model = walk(step.verify, iter_context)
|
|
417
504
|
cls._process_verify_step(verify_model, response)
|
|
418
505
|
except StageExecutionError as e:
|
|
419
506
|
e.request = response.request
|
|
@@ -542,7 +629,11 @@ def create_test_class(scenario: Scenario, class_name: str, max_parallel_iteratio
|
|
|
542
629
|
try:
|
|
543
630
|
stage_method = make_marker(mark_str)(stage_method)
|
|
544
631
|
except Exception as e:
|
|
545
|
-
|
|
632
|
+
# A malformed stage marker is an author error: fail collection (the
|
|
633
|
+
# caller wraps this into a CollectError) instead of silently dropping
|
|
634
|
+
# the marker and running the stage — matching how scenario-level
|
|
635
|
+
# markers are handled in plugin.py.
|
|
636
|
+
raise StageExecutionError(f"Invalid marker '{mark_str}' on stage '{stage.name}': {e}") from e
|
|
546
637
|
|
|
547
638
|
method_name = f"test {str(i).zfill(padding_width)} - {stage.name}"
|
|
548
639
|
setattr(CustomCarrier, method_name, stage_method)
|