tenzir-test 0.9.5__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tenzir_test/__init__.py +6 -0
- tenzir_test/cli.py +31 -24
- tenzir_test/run.py +205 -44
- tenzir_test/runners/shell_runner.py +28 -26
- {tenzir_test-0.9.5.dist-info → tenzir_test-0.11.0.dist-info}/METADATA +1 -1
- {tenzir_test-0.9.5.dist-info → tenzir_test-0.11.0.dist-info}/RECORD +9 -9
- {tenzir_test-0.9.5.dist-info → tenzir_test-0.11.0.dist-info}/WHEEL +0 -0
- {tenzir_test-0.9.5.dist-info → tenzir_test-0.11.0.dist-info}/entry_points.txt +0 -0
- {tenzir_test-0.9.5.dist-info → tenzir_test-0.11.0.dist-info}/licenses/LICENSE +0 -0
tenzir_test/__init__.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from importlib.metadata import PackageNotFoundError, version
|
|
4
4
|
|
|
5
5
|
from . import run
|
|
6
|
+
from .run import ExecutionResult, HarnessError, ProjectResult, ensure_settings, execute
|
|
6
7
|
from .config import Settings, discover_settings
|
|
7
8
|
from .fixtures import (
|
|
8
9
|
Executor,
|
|
@@ -32,9 +33,14 @@ __all__ = [
|
|
|
32
33
|
"fixtures",
|
|
33
34
|
"Settings",
|
|
34
35
|
"discover_settings",
|
|
36
|
+
"ensure_settings",
|
|
37
|
+
"execute",
|
|
35
38
|
"has",
|
|
36
39
|
"register",
|
|
37
40
|
"require",
|
|
41
|
+
"ExecutionResult",
|
|
42
|
+
"ProjectResult",
|
|
43
|
+
"HarnessError",
|
|
38
44
|
"run",
|
|
39
45
|
]
|
|
40
46
|
|
tenzir_test/cli.py
CHANGED
|
@@ -152,33 +152,39 @@ def cli(
|
|
|
152
152
|
jobs: int,
|
|
153
153
|
passthrough: bool,
|
|
154
154
|
all_projects: bool,
|
|
155
|
-
) ->
|
|
155
|
+
) -> int:
|
|
156
156
|
"""Execute tenzir-test scenarios."""
|
|
157
157
|
|
|
158
158
|
jobs_source = ctx.get_parameter_source("jobs")
|
|
159
159
|
jobs_overridden = jobs_source is not click.core.ParameterSource.DEFAULT
|
|
160
160
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
161
|
+
try:
|
|
162
|
+
result = runtime.run_cli(
|
|
163
|
+
root=root,
|
|
164
|
+
tenzir_binary=tenzir_binary,
|
|
165
|
+
tenzir_node_binary=tenzir_node_binary,
|
|
166
|
+
tests=list(tests),
|
|
167
|
+
update=update,
|
|
168
|
+
debug=debug,
|
|
169
|
+
purge=purge,
|
|
170
|
+
coverage=coverage,
|
|
171
|
+
coverage_source_dir=coverage_source_dir,
|
|
172
|
+
runner_summary=runner_summary,
|
|
173
|
+
fixture_summary=fixture_summary,
|
|
174
|
+
show_summary=show_summary,
|
|
175
|
+
show_diff_output=show_diff_output,
|
|
176
|
+
show_diff_stat=show_diff_stat,
|
|
177
|
+
keep_tmp_dirs=keep_tmp_dirs,
|
|
178
|
+
jobs=jobs,
|
|
179
|
+
passthrough=passthrough,
|
|
180
|
+
jobs_overridden=jobs_overridden,
|
|
181
|
+
all_projects=all_projects,
|
|
182
|
+
)
|
|
183
|
+
except runtime.HarnessError as exc:
|
|
184
|
+
if exc.show_message and exc.args:
|
|
185
|
+
raise click.ClickException(str(exc)) from exc
|
|
186
|
+
return exc.exit_code
|
|
187
|
+
return result.exit_code
|
|
182
188
|
|
|
183
189
|
|
|
184
190
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
@@ -186,7 +192,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
186
192
|
|
|
187
193
|
command_main = getattr(cli, "main")
|
|
188
194
|
try:
|
|
189
|
-
command_main(
|
|
195
|
+
result = command_main(
|
|
190
196
|
args=list(argv) if argv is not None else None,
|
|
191
197
|
standalone_mode=False,
|
|
192
198
|
)
|
|
@@ -198,7 +204,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
198
204
|
return _normalize_exit_code(exit_code)
|
|
199
205
|
except SystemExit as exc: # pragma: no cover - propagate runner exits
|
|
200
206
|
return _normalize_exit_code(exc.code)
|
|
201
|
-
|
|
207
|
+
else:
|
|
208
|
+
return _normalize_exit_code(result)
|
|
202
209
|
|
|
203
210
|
|
|
204
211
|
if __name__ == "__main__":
|
tenzir_test/run.py
CHANGED
|
@@ -151,12 +151,13 @@ def detect_execution_mode(root: Path) -> tuple[ExecutionMode, Path | None]:
|
|
|
151
151
|
return ExecutionMode.PROJECT, None
|
|
152
152
|
|
|
153
153
|
|
|
154
|
-
_settings =
|
|
155
|
-
TENZIR_BINARY =
|
|
156
|
-
TENZIR_NODE_BINARY =
|
|
157
|
-
ROOT =
|
|
158
|
-
INPUTS_DIR =
|
|
159
|
-
EXECUTION_MODE
|
|
154
|
+
_settings: Settings | None = None
|
|
155
|
+
TENZIR_BINARY: str | None = None
|
|
156
|
+
TENZIR_NODE_BINARY: str | None = None
|
|
157
|
+
ROOT: Path = Path.cwd()
|
|
158
|
+
INPUTS_DIR: Path = ROOT / "inputs"
|
|
159
|
+
EXECUTION_MODE: ExecutionMode = ExecutionMode.PROJECT
|
|
160
|
+
_DETECTED_PACKAGE_ROOT: Path | None = None
|
|
160
161
|
HARNESS_MODE = HarnessMode.COMPARE
|
|
161
162
|
CHECKMARK = "\033[92;1m✔\033[0m"
|
|
162
163
|
CROSS = "\033[31m✘\033[0m"
|
|
@@ -169,6 +170,7 @@ PASS_MAX_COLOR = "\033[92m"
|
|
|
169
170
|
FAIL_COLOR = "\033[31m"
|
|
170
171
|
SKIP_COLOR = "\033[90;1m"
|
|
171
172
|
RESET_COLOR = "\033[0m"
|
|
173
|
+
DETAIL_COLOR = "\033[2;37m"
|
|
172
174
|
PASS_SPECTRUM = [
|
|
173
175
|
"\033[38;5;52m", # 0-9% deep red
|
|
174
176
|
"\033[38;5;88m", # 10-19% red
|
|
@@ -1458,6 +1460,14 @@ def _looks_like_project_root(path: Path) -> bool:
|
|
|
1458
1460
|
return False
|
|
1459
1461
|
|
|
1460
1462
|
|
|
1463
|
+
def ensure_settings() -> Settings:
|
|
1464
|
+
"""Return the active harness settings, discovering defaults on first use."""
|
|
1465
|
+
|
|
1466
|
+
if _settings is None:
|
|
1467
|
+
apply_settings(discover_settings())
|
|
1468
|
+
return cast(Settings, _settings)
|
|
1469
|
+
|
|
1470
|
+
|
|
1461
1471
|
def apply_settings(settings: Settings) -> None:
|
|
1462
1472
|
global TENZIR_BINARY, TENZIR_NODE_BINARY
|
|
1463
1473
|
global _settings
|
|
@@ -1825,6 +1835,31 @@ class Summary:
|
|
|
1825
1835
|
stats.failed += 1
|
|
1826
1836
|
|
|
1827
1837
|
|
|
1838
|
+
@dataclasses.dataclass(slots=True)
|
|
1839
|
+
class ProjectResult:
|
|
1840
|
+
selection: ProjectSelection
|
|
1841
|
+
summary: Summary
|
|
1842
|
+
queue_size: int
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
@dataclasses.dataclass(slots=True)
|
|
1846
|
+
class ExecutionResult:
|
|
1847
|
+
summary: Summary
|
|
1848
|
+
project_results: tuple[ProjectResult, ...]
|
|
1849
|
+
queue_size: int
|
|
1850
|
+
exit_code: int
|
|
1851
|
+
interrupted: bool
|
|
1852
|
+
|
|
1853
|
+
|
|
1854
|
+
class HarnessError(RuntimeError):
|
|
1855
|
+
"""Fatal harness error signalling invalid invocation or configuration."""
|
|
1856
|
+
|
|
1857
|
+
def __init__(self, message: str, *, exit_code: int = 1, show_message: bool = True) -> None:
|
|
1858
|
+
super().__init__(message)
|
|
1859
|
+
self.exit_code = exit_code
|
|
1860
|
+
self.show_message = show_message
|
|
1861
|
+
|
|
1862
|
+
|
|
1828
1863
|
def _format_percentage(count: int, total: int) -> str:
|
|
1829
1864
|
return f"{_percentage_value(count, total)}%"
|
|
1830
1865
|
|
|
@@ -1918,7 +1953,7 @@ def _build_queue_from_paths(
|
|
|
1918
1953
|
try:
|
|
1919
1954
|
runner = get_runner_for_test(test_path)
|
|
1920
1955
|
except ValueError as error:
|
|
1921
|
-
|
|
1956
|
+
raise HarnessError(f"error: {error}") from error
|
|
1922
1957
|
|
|
1923
1958
|
suite_info = _resolve_suite_for_test(test_path)
|
|
1924
1959
|
test_item = TestQueueItem(runner=runner, path=test_path)
|
|
@@ -1950,7 +1985,7 @@ def _build_queue_from_paths(
|
|
|
1950
1985
|
location_detail = (
|
|
1951
1986
|
f" ({_relativize_path(mismatch_path)})" if mismatch_path is not None else ""
|
|
1952
1987
|
)
|
|
1953
|
-
|
|
1988
|
+
raise HarnessError(
|
|
1954
1989
|
f"error: suite '{suite_info.name}' defined in {config_path} must use identical fixtures "
|
|
1955
1990
|
f"across tests (expected: {expected_list}, found: {example_list}{location_detail})"
|
|
1956
1991
|
)
|
|
@@ -2632,9 +2667,13 @@ def run_simple_test(
|
|
|
2632
2667
|
)
|
|
2633
2668
|
good = completed.returncode == 0
|
|
2634
2669
|
output = b""
|
|
2670
|
+
stderr_output = b""
|
|
2635
2671
|
if not passthrough_mode:
|
|
2636
|
-
|
|
2637
|
-
|
|
2672
|
+
root_bytes = str(ROOT).encode() + b"/"
|
|
2673
|
+
captured_stdout = completed.stdout or b""
|
|
2674
|
+
output = captured_stdout.replace(root_bytes, b"")
|
|
2675
|
+
captured_stderr = completed.stderr or b""
|
|
2676
|
+
stderr_output = captured_stderr.replace(root_bytes, b"")
|
|
2638
2677
|
except subprocess.TimeoutExpired:
|
|
2639
2678
|
report_failure(test, f"└─▶ \033[31msubprocess hit {timeout}s timeout\033[0m")
|
|
2640
2679
|
return False
|
|
@@ -2654,20 +2693,30 @@ def run_simple_test(
|
|
|
2654
2693
|
return False
|
|
2655
2694
|
if interrupted:
|
|
2656
2695
|
_request_interrupt()
|
|
2657
|
-
|
|
2696
|
+
summary_line = (
|
|
2658
2697
|
_INTERRUPTED_NOTICE
|
|
2659
2698
|
if interrupted
|
|
2660
|
-
else f"
|
|
2699
|
+
else f"└─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m"
|
|
2661
2700
|
)
|
|
2662
2701
|
if passthrough_mode:
|
|
2663
|
-
report_failure(test,
|
|
2702
|
+
report_failure(test, summary_line)
|
|
2664
2703
|
else:
|
|
2665
2704
|
with stdout_lock:
|
|
2666
|
-
|
|
2705
|
+
fail(test)
|
|
2667
2706
|
if not interrupted:
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
sys.stdout.buffer.write(
|
|
2707
|
+
line_prefix = "│ ".encode()
|
|
2708
|
+
for line in output.splitlines():
|
|
2709
|
+
sys.stdout.buffer.write(line_prefix + line + b"\n")
|
|
2710
|
+
if completed.returncode != 0 and stderr_output:
|
|
2711
|
+
sys.stdout.write("├─▶ stderr\n")
|
|
2712
|
+
detail_prefix = DETAIL_COLOR.encode()
|
|
2713
|
+
reset_bytes = RESET_COLOR.encode()
|
|
2714
|
+
for line in stderr_output.splitlines():
|
|
2715
|
+
sys.stdout.buffer.write(
|
|
2716
|
+
line_prefix + detail_prefix + line + reset_bytes + b"\n"
|
|
2717
|
+
)
|
|
2718
|
+
if summary_line:
|
|
2719
|
+
sys.stdout.write(summary_line + "\n")
|
|
2671
2720
|
return False
|
|
2672
2721
|
if passthrough_mode:
|
|
2673
2722
|
success(test)
|
|
@@ -3003,7 +3052,8 @@ def run_cli(
|
|
|
3003
3052
|
passthrough: bool,
|
|
3004
3053
|
jobs_overridden: bool = False,
|
|
3005
3054
|
all_projects: bool = False,
|
|
3006
|
-
) ->
|
|
3055
|
+
) -> ExecutionResult:
|
|
3056
|
+
"""Execute the harness and return a structured result for library consumers."""
|
|
3007
3057
|
from tenzir_test.engine import state as engine_state
|
|
3008
3058
|
|
|
3009
3059
|
try:
|
|
@@ -3083,25 +3133,31 @@ def run_cli(
|
|
|
3083
3133
|
|
|
3084
3134
|
if not _is_project_root(ROOT):
|
|
3085
3135
|
if all_projects:
|
|
3086
|
-
|
|
3136
|
+
raise HarnessError(
|
|
3137
|
+
"error: --all-projects requires a project root; specify one with --root"
|
|
3138
|
+
)
|
|
3087
3139
|
if not selected_tests:
|
|
3088
|
-
|
|
3140
|
+
message = (
|
|
3089
3141
|
f"{INFO} no tenzir-test project detected at {ROOT}.\n"
|
|
3090
3142
|
f"{INFO} run from your project root or provide --root."
|
|
3091
3143
|
)
|
|
3092
|
-
|
|
3144
|
+
print(message)
|
|
3145
|
+
raise HarnessError(message, show_message=False)
|
|
3093
3146
|
assert plan is not None
|
|
3094
3147
|
runnable_satellites = [item for item in plan.satellites if item.should_run()]
|
|
3095
3148
|
if not runnable_satellites:
|
|
3096
|
-
|
|
3149
|
+
message = (
|
|
3097
3150
|
f"{INFO} no tenzir-test project detected at {ROOT}.\n"
|
|
3098
3151
|
f"{INFO} run from your project root or provide --root."
|
|
3099
3152
|
)
|
|
3153
|
+
print(message)
|
|
3100
3154
|
sample = ", ".join(str(path) for path in selected_tests[:3])
|
|
3101
3155
|
if len(selected_tests) > 3:
|
|
3102
3156
|
sample += ", ..."
|
|
3103
3157
|
print(f"{INFO} ignoring provided selection(s): {sample}")
|
|
3104
|
-
|
|
3158
|
+
raise HarnessError(
|
|
3159
|
+
"no runnable tests selected outside of a project root", show_message=False
|
|
3160
|
+
)
|
|
3105
3161
|
if plan is None:
|
|
3106
3162
|
plan = _build_execution_plan(
|
|
3107
3163
|
ROOT,
|
|
@@ -3120,7 +3176,9 @@ def run_cli(
|
|
|
3120
3176
|
overall_summary = Summary()
|
|
3121
3177
|
overall_queue_count = 0
|
|
3122
3178
|
executed_projects: list[ProjectSelection] = []
|
|
3179
|
+
project_results: list[ProjectResult] = []
|
|
3123
3180
|
printed_projects = 0
|
|
3181
|
+
interrupted = False
|
|
3124
3182
|
|
|
3125
3183
|
for selection in plan.projects():
|
|
3126
3184
|
if interrupt_requested():
|
|
@@ -3133,7 +3191,7 @@ def run_cli(
|
|
|
3133
3191
|
_load_project_runners(selection.root, expose_namespace=True)
|
|
3134
3192
|
_load_project_fixtures(selection.root, expose_namespace=True)
|
|
3135
3193
|
except RuntimeError as exc:
|
|
3136
|
-
|
|
3194
|
+
raise HarnessError(f"error: {exc}") from exc
|
|
3137
3195
|
refresh_runner_metadata()
|
|
3138
3196
|
_set_project_root(settings.root)
|
|
3139
3197
|
engine_state.refresh()
|
|
@@ -3149,7 +3207,7 @@ def run_cli(
|
|
|
3149
3207
|
_load_project_runners(selection.root, expose_namespace=expose_namespace)
|
|
3150
3208
|
_load_project_fixtures(selection.root, expose_namespace=expose_namespace)
|
|
3151
3209
|
except RuntimeError as exc:
|
|
3152
|
-
|
|
3210
|
+
raise HarnessError(f"error: {exc}") from exc
|
|
3153
3211
|
refresh_runner_metadata()
|
|
3154
3212
|
|
|
3155
3213
|
tests_to_run = selection.selectors if not selection.run_all else [selection.root]
|
|
@@ -3169,14 +3227,16 @@ def run_cli(
|
|
|
3169
3227
|
|
|
3170
3228
|
resolved = test.resolve()
|
|
3171
3229
|
if not resolved.exists():
|
|
3172
|
-
|
|
3230
|
+
raise HarnessError(f"error: test path `{test}` does not exist")
|
|
3173
3231
|
|
|
3174
3232
|
if resolved.is_dir():
|
|
3175
3233
|
if _is_inputs_path(resolved):
|
|
3176
3234
|
continue
|
|
3177
3235
|
tql_files = list(collect_all_tests(resolved))
|
|
3178
3236
|
if not tql_files:
|
|
3179
|
-
|
|
3237
|
+
raise HarnessError(
|
|
3238
|
+
f"error: no {_allowed_extensions} files found in {resolved}"
|
|
3239
|
+
)
|
|
3180
3240
|
for file_path in tql_files:
|
|
3181
3241
|
suite_info = _resolve_suite_for_test(file_path)
|
|
3182
3242
|
if suite_info is None:
|
|
@@ -3196,7 +3256,10 @@ def run_cli(
|
|
|
3196
3256
|
f"{INFO} select the suite directory instead",
|
|
3197
3257
|
file=sys.stderr,
|
|
3198
3258
|
)
|
|
3199
|
-
|
|
3259
|
+
raise HarnessError(
|
|
3260
|
+
f"invalid partial suite selection at {rel_target}",
|
|
3261
|
+
show_message=False,
|
|
3262
|
+
)
|
|
3200
3263
|
for file_path in tql_files:
|
|
3201
3264
|
collected_paths.add(file_path.resolve())
|
|
3202
3265
|
elif resolved.is_file():
|
|
@@ -3214,18 +3277,18 @@ def run_cli(
|
|
|
3214
3277
|
f"'{suite_info.name}' defined in {rel_suite / _CONFIG_FILE_NAME}."
|
|
3215
3278
|
)
|
|
3216
3279
|
print(f"{CROSS} {detail}", file=sys.stderr)
|
|
3217
|
-
print(
|
|
3218
|
-
|
|
3219
|
-
|
|
3280
|
+
print(f"{INFO} select the suite directory instead", file=sys.stderr)
|
|
3281
|
+
raise HarnessError(
|
|
3282
|
+
f"invalid suite selection for {rel_file}",
|
|
3283
|
+
show_message=False,
|
|
3220
3284
|
)
|
|
3221
|
-
sys.exit(1)
|
|
3222
3285
|
collected_paths.add(resolved.resolve())
|
|
3223
3286
|
else:
|
|
3224
|
-
|
|
3287
|
+
raise HarnessError(
|
|
3225
3288
|
f"error: unsupported file type {resolved.suffix} for {resolved} - only {_allowed_extensions} files are supported"
|
|
3226
3289
|
)
|
|
3227
3290
|
else:
|
|
3228
|
-
|
|
3291
|
+
raise HarnessError(f"error: `{test}` is neither a file nor a directory")
|
|
3229
3292
|
|
|
3230
3293
|
if interrupt_requested():
|
|
3231
3294
|
break
|
|
@@ -3233,6 +3296,7 @@ def run_cli(
|
|
|
3233
3296
|
queue = _build_queue_from_paths(collected_paths, coverage=coverage)
|
|
3234
3297
|
queue.sort(key=_queue_sort_key, reverse=True)
|
|
3235
3298
|
project_queue_size = _count_queue_tests(queue)
|
|
3299
|
+
project_summary = Summary()
|
|
3236
3300
|
job_count, enabled_flags, verb = _summarize_harness_configuration(
|
|
3237
3301
|
jobs=jobs,
|
|
3238
3302
|
update=update,
|
|
@@ -3247,15 +3311,26 @@ def run_cli(
|
|
|
3247
3311
|
if not project_queue_size:
|
|
3248
3312
|
overall_queue_count += project_queue_size
|
|
3249
3313
|
executed_projects.append(selection)
|
|
3314
|
+
project_results.append(
|
|
3315
|
+
ProjectResult(
|
|
3316
|
+
selection=selection,
|
|
3317
|
+
summary=project_summary,
|
|
3318
|
+
queue_size=project_queue_size,
|
|
3319
|
+
)
|
|
3320
|
+
)
|
|
3250
3321
|
continue
|
|
3251
3322
|
|
|
3252
3323
|
os.environ["TENZIR_EXEC__DUMP_DIAGNOSTICS"] = "true"
|
|
3253
3324
|
if not TENZIR_BINARY:
|
|
3254
|
-
|
|
3325
|
+
raise HarnessError(
|
|
3326
|
+
f"error: could not find TENZIR_BINARY executable `{TENZIR_BINARY}`"
|
|
3327
|
+
)
|
|
3255
3328
|
try:
|
|
3256
3329
|
tenzir_version = get_version()
|
|
3257
3330
|
except FileNotFoundError:
|
|
3258
|
-
|
|
3331
|
+
raise HarnessError(
|
|
3332
|
+
f"error: could not find TENZIR_BINARY executable `{TENZIR_BINARY}`"
|
|
3333
|
+
)
|
|
3259
3334
|
|
|
3260
3335
|
runner_versions = _collect_runner_versions(queue, tenzir_version=tenzir_version)
|
|
3261
3336
|
runner_breakdown = _runner_breakdown(
|
|
@@ -3288,7 +3363,6 @@ def run_cli(
|
|
|
3288
3363
|
)
|
|
3289
3364
|
for _ in range(jobs)
|
|
3290
3365
|
]
|
|
3291
|
-
project_summary = Summary()
|
|
3292
3366
|
for worker in workers:
|
|
3293
3367
|
worker.start()
|
|
3294
3368
|
try:
|
|
@@ -3298,6 +3372,7 @@ def run_cli(
|
|
|
3298
3372
|
_request_interrupt()
|
|
3299
3373
|
for worker in workers:
|
|
3300
3374
|
worker.join()
|
|
3375
|
+
interrupted = True
|
|
3301
3376
|
break
|
|
3302
3377
|
|
|
3303
3378
|
_print_compact_summary(project_summary)
|
|
@@ -3321,6 +3396,13 @@ def run_cli(
|
|
|
3321
3396
|
overall_summary += project_summary
|
|
3322
3397
|
overall_queue_count += project_queue_size
|
|
3323
3398
|
executed_projects.append(selection)
|
|
3399
|
+
project_results.append(
|
|
3400
|
+
ProjectResult(
|
|
3401
|
+
selection=selection,
|
|
3402
|
+
summary=project_summary,
|
|
3403
|
+
queue_size=project_queue_size,
|
|
3404
|
+
)
|
|
3405
|
+
)
|
|
3324
3406
|
|
|
3325
3407
|
if interrupt_requested():
|
|
3326
3408
|
break
|
|
@@ -3332,39 +3414,118 @@ def run_cli(
|
|
|
3332
3414
|
if purge:
|
|
3333
3415
|
for runner in runners_iter_runners():
|
|
3334
3416
|
runner.purge()
|
|
3335
|
-
return
|
|
3417
|
+
return ExecutionResult(
|
|
3418
|
+
summary=overall_summary,
|
|
3419
|
+
project_results=tuple(project_results),
|
|
3420
|
+
queue_size=overall_queue_count,
|
|
3421
|
+
exit_code=0,
|
|
3422
|
+
interrupted=interrupted,
|
|
3423
|
+
)
|
|
3336
3424
|
|
|
3337
3425
|
if overall_queue_count == 0:
|
|
3338
3426
|
print(f"{INFO} no tests selected")
|
|
3339
|
-
return
|
|
3427
|
+
return ExecutionResult(
|
|
3428
|
+
summary=overall_summary,
|
|
3429
|
+
project_results=tuple(project_results),
|
|
3430
|
+
queue_size=overall_queue_count,
|
|
3431
|
+
exit_code=0,
|
|
3432
|
+
interrupted=interrupted,
|
|
3433
|
+
)
|
|
3340
3434
|
|
|
3341
3435
|
if len(executed_projects) > 1:
|
|
3342
3436
|
_print_aggregate_totals(len(executed_projects), overall_summary)
|
|
3343
3437
|
|
|
3344
|
-
if
|
|
3345
|
-
|
|
3438
|
+
if interrupted:
|
|
3439
|
+
return ExecutionResult(
|
|
3440
|
+
summary=overall_summary,
|
|
3441
|
+
project_results=tuple(project_results),
|
|
3442
|
+
queue_size=overall_queue_count,
|
|
3443
|
+
exit_code=130,
|
|
3444
|
+
interrupted=True,
|
|
3445
|
+
)
|
|
3346
3446
|
|
|
3347
|
-
if overall_summary.failed > 0
|
|
3348
|
-
|
|
3447
|
+
exit_code = 1 if overall_summary.failed > 0 else 0
|
|
3448
|
+
return ExecutionResult(
|
|
3449
|
+
summary=overall_summary,
|
|
3450
|
+
project_results=tuple(project_results),
|
|
3451
|
+
queue_size=overall_queue_count,
|
|
3452
|
+
exit_code=exit_code,
|
|
3453
|
+
interrupted=False,
|
|
3454
|
+
)
|
|
3349
3455
|
|
|
3350
3456
|
finally:
|
|
3351
3457
|
_cleanup_all_tmp_dirs()
|
|
3352
3458
|
|
|
3353
3459
|
|
|
3460
|
+
def execute(
|
|
3461
|
+
*,
|
|
3462
|
+
root: Path | None = None,
|
|
3463
|
+
tenzir_binary: Path | None = None,
|
|
3464
|
+
tenzir_node_binary: Path | None = None,
|
|
3465
|
+
tests: Sequence[Path] = (),
|
|
3466
|
+
update: bool = False,
|
|
3467
|
+
debug: bool = False,
|
|
3468
|
+
purge: bool = False,
|
|
3469
|
+
coverage: bool = False,
|
|
3470
|
+
coverage_source_dir: Path | None = None,
|
|
3471
|
+
runner_summary: bool = False,
|
|
3472
|
+
fixture_summary: bool = False,
|
|
3473
|
+
show_summary: bool = False,
|
|
3474
|
+
show_diff_output: bool = True,
|
|
3475
|
+
show_diff_stat: bool = True,
|
|
3476
|
+
jobs: int | None = None,
|
|
3477
|
+
keep_tmp_dirs: bool = False,
|
|
3478
|
+
passthrough: bool = False,
|
|
3479
|
+
jobs_overridden: bool = False,
|
|
3480
|
+
all_projects: bool = False,
|
|
3481
|
+
) -> ExecutionResult:
|
|
3482
|
+
"""Library-oriented wrapper around `run_cli` with defaulted parameters."""
|
|
3483
|
+
|
|
3484
|
+
resolved_jobs = jobs if jobs is not None else get_default_jobs()
|
|
3485
|
+
return run_cli(
|
|
3486
|
+
root=root,
|
|
3487
|
+
tenzir_binary=tenzir_binary,
|
|
3488
|
+
tenzir_node_binary=tenzir_node_binary,
|
|
3489
|
+
tests=list(tests),
|
|
3490
|
+
update=update,
|
|
3491
|
+
debug=debug,
|
|
3492
|
+
purge=purge,
|
|
3493
|
+
coverage=coverage,
|
|
3494
|
+
coverage_source_dir=coverage_source_dir,
|
|
3495
|
+
runner_summary=runner_summary,
|
|
3496
|
+
fixture_summary=fixture_summary,
|
|
3497
|
+
show_summary=show_summary,
|
|
3498
|
+
show_diff_output=show_diff_output,
|
|
3499
|
+
show_diff_stat=show_diff_stat,
|
|
3500
|
+
jobs=resolved_jobs,
|
|
3501
|
+
keep_tmp_dirs=keep_tmp_dirs,
|
|
3502
|
+
passthrough=passthrough,
|
|
3503
|
+
jobs_overridden=jobs_overridden,
|
|
3504
|
+
all_projects=all_projects,
|
|
3505
|
+
)
|
|
3506
|
+
|
|
3507
|
+
|
|
3354
3508
|
def main(argv: Sequence[str] | None = None) -> None:
|
|
3355
3509
|
import click
|
|
3356
3510
|
|
|
3357
3511
|
from . import cli as cli_module
|
|
3358
3512
|
|
|
3359
3513
|
try:
|
|
3360
|
-
cli_module.cli.main(
|
|
3514
|
+
result = cli_module.cli.main(
|
|
3361
3515
|
args=list(argv) if argv is not None else None,
|
|
3362
3516
|
standalone_mode=False,
|
|
3363
3517
|
)
|
|
3518
|
+
except click.exceptions.ClickException as exc:
|
|
3519
|
+
exc.show(file=sys.stderr)
|
|
3520
|
+
exit_code = getattr(exc, "exit_code", 1)
|
|
3521
|
+
raise SystemExit(exit_code) from exc
|
|
3364
3522
|
except click.exceptions.Exit as exc:
|
|
3365
3523
|
raise SystemExit(exc.exit_code) from exc
|
|
3366
3524
|
except click.exceptions.Abort as exc:
|
|
3367
3525
|
raise SystemExit(1) from exc
|
|
3526
|
+
exit_code = cli_module._normalize_exit_code(result)
|
|
3527
|
+
if exit_code:
|
|
3528
|
+
raise SystemExit(exit_code)
|
|
3368
3529
|
|
|
3369
3530
|
|
|
3370
3531
|
if __name__ == "__main__":
|
|
@@ -64,47 +64,49 @@ class ShellRunner(ExtRunner):
|
|
|
64
64
|
fixture_api.pop_context(context_token)
|
|
65
65
|
run_mod.cleanup_test_tmp_dir(env.get(run_mod.TEST_TMP_ENV_VAR))
|
|
66
66
|
|
|
67
|
+
stdout_data = completed.stdout
|
|
68
|
+
if isinstance(stdout_data, str):
|
|
69
|
+
stdout_bytes: bytes = stdout_data.encode()
|
|
70
|
+
else:
|
|
71
|
+
stdout_bytes = stdout_data or b""
|
|
72
|
+
|
|
73
|
+
stderr_data = completed.stderr
|
|
74
|
+
if isinstance(stderr_data, str):
|
|
75
|
+
stderr_bytes: bytes = stderr_data.encode()
|
|
76
|
+
else:
|
|
77
|
+
stderr_bytes = stderr_data or b""
|
|
78
|
+
|
|
67
79
|
good = completed.returncode == 0
|
|
68
80
|
if expect_error == good:
|
|
69
81
|
suppressed = run_mod.should_suppress_failure_output()
|
|
82
|
+
summary_line = f"└─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m"
|
|
70
83
|
if passthrough:
|
|
71
84
|
if not suppressed:
|
|
72
|
-
run_mod.report_failure(
|
|
73
|
-
test,
|
|
74
|
-
f"┌─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m",
|
|
75
|
-
)
|
|
85
|
+
run_mod.report_failure(test, summary_line)
|
|
76
86
|
return False
|
|
77
87
|
if suppressed:
|
|
78
88
|
return False
|
|
89
|
+
|
|
79
90
|
with run_mod.stdout_lock:
|
|
80
|
-
run_mod.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
91
|
+
run_mod.fail(test)
|
|
92
|
+
line_prefix = "│ ".encode()
|
|
93
|
+
for line in stdout_bytes.splitlines():
|
|
94
|
+
sys.stdout.buffer.write(line_prefix + line + b"\n")
|
|
95
|
+
if stderr_bytes:
|
|
96
|
+
sys.stdout.write("├─▶ stderr\n")
|
|
97
|
+
detail_prefix = run_mod.DETAIL_COLOR.encode()
|
|
98
|
+
reset_bytes = run_mod.RESET_COLOR.encode()
|
|
99
|
+
for line in stderr_bytes.splitlines():
|
|
100
|
+
sys.stdout.buffer.write(
|
|
101
|
+
line_prefix + detail_prefix + line + reset_bytes + b"\n"
|
|
102
|
+
)
|
|
103
|
+
sys.stdout.write(summary_line + "\n")
|
|
90
104
|
return False
|
|
91
105
|
|
|
92
106
|
if passthrough:
|
|
93
107
|
run_mod.success(test)
|
|
94
108
|
return True
|
|
95
109
|
|
|
96
|
-
stdout_data = completed.stdout
|
|
97
|
-
if isinstance(stdout_data, str):
|
|
98
|
-
stdout_bytes: bytes = stdout_data.encode()
|
|
99
|
-
else:
|
|
100
|
-
stdout_bytes = stdout_data or b""
|
|
101
|
-
|
|
102
|
-
stderr_data = completed.stderr
|
|
103
|
-
if isinstance(stderr_data, str):
|
|
104
|
-
stderr_bytes: bytes = stderr_data.encode()
|
|
105
|
-
else:
|
|
106
|
-
stderr_bytes = stderr_data or b""
|
|
107
|
-
|
|
108
110
|
root_prefix: bytes = (str(run_mod.ROOT) + "/").encode()
|
|
109
111
|
stdout_bytes = stdout_bytes.replace(root_prefix, b"")
|
|
110
112
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tenzir-test
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Reusable test execution framework extracted from the Tenzir repository.
|
|
5
5
|
Project-URL: Homepage, https://github.com/tenzir/test
|
|
6
6
|
Project-URL: Repository, https://github.com/tenzir/test
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
tenzir_test/__init__.py,sha256=
|
|
1
|
+
tenzir_test/__init__.py,sha256=k7V6Pbjaa8SAy6t4KnaauHTyfnyVEwc1VGtH823MANU,1181
|
|
2
2
|
tenzir_test/_python_runner.py,sha256=LmghMIolsNEC2wUyJdv1h_cefOxTxET1IACrw-_hHuY,2900
|
|
3
3
|
tenzir_test/checks.py,sha256=VhZjU1TExqWzA1KcaW1xOGICpqb_G43AezrJIzw09eM,653
|
|
4
|
-
tenzir_test/cli.py,sha256=
|
|
4
|
+
tenzir_test/cli.py,sha256=aTNKPscOfZtrMLoIHV-8NcmngXy4HSFGP_-mM0gK7CE,5800
|
|
5
5
|
tenzir_test/config.py,sha256=q1_VEXuxL-xsGlnooeGvXxx9cMw652UEB9a1mPzZIQs,1680
|
|
6
6
|
tenzir_test/packages.py,sha256=cTCQdGjCS1XmuKyiwh0ew-z9tHn6J-xZ6nvBP-hU8bc,948
|
|
7
7
|
tenzir_test/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
tenzir_test/run.py,sha256=
|
|
8
|
+
tenzir_test/run.py,sha256=mDLGcIbG9AqfER86BmGRzt4WMCh322OkdKJKHtTUjDw,122173
|
|
9
9
|
tenzir_test/engine/__init__.py,sha256=5APwy90YDm7rmL_qCZfToAcfbQthcZ8yV2_ExXKqaqE,110
|
|
10
10
|
tenzir_test/engine/operations.py,sha256=OCYjuMHyMAaay4s08u2Sl7oE-PmgeXumylp7R8GYIH4,950
|
|
11
11
|
tenzir_test/engine/registry.py,sha256=LXCr6TGlv1sR1m1eboTk7SrbS2IVErc3PqUuHxGA2xk,594
|
|
@@ -19,11 +19,11 @@ tenzir_test/runners/custom_python_fixture_runner.py,sha256=bbG0ZvCL6Cs6BCKJQQPAB
|
|
|
19
19
|
tenzir_test/runners/diff_runner.py,sha256=ah1hr1vvD6BON2PZz61mxwioRFIzHFuaAbJ0DjDSqG4,5151
|
|
20
20
|
tenzir_test/runners/ext_runner.py,sha256=sKL9Mw_ksVVBWnrdIJR2WS5ueVnLKuNYYWZ22FTZIPo,730
|
|
21
21
|
tenzir_test/runners/runner.py,sha256=LtlD8huQOSmD7RyYDnKeCuI4Y6vhxGXMKsHA2qgfWN0,989
|
|
22
|
-
tenzir_test/runners/shell_runner.py,sha256=
|
|
22
|
+
tenzir_test/runners/shell_runner.py,sha256=EREqIaHxG5_nl8CmeQYWsiM6rZS9frCkhdaUHoRUGRw,6024
|
|
23
23
|
tenzir_test/runners/tenzir_runner.py,sha256=464FFYS_mh6l-ehccc-S8cIUO1MxdapwQL5X3PmMkMI,1006
|
|
24
24
|
tenzir_test/runners/tql_runner.py,sha256=2ZLMf3TIKwcOvaOFrVvvhzK-EcWmGOUZxKkbSoByyQA,248
|
|
25
|
-
tenzir_test-0.
|
|
26
|
-
tenzir_test-0.
|
|
27
|
-
tenzir_test-0.
|
|
28
|
-
tenzir_test-0.
|
|
29
|
-
tenzir_test-0.
|
|
25
|
+
tenzir_test-0.11.0.dist-info/METADATA,sha256=lNG4jTPWJX7biDdS0ZL1TVh8VaJv96dIXCX1Ab6NzPE,3008
|
|
26
|
+
tenzir_test-0.11.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
27
|
+
tenzir_test-0.11.0.dist-info/entry_points.txt,sha256=q0eD9RQ_9eMPYvFNpBElo55HQYeaPgLfe9YhLsNwl10,93
|
|
28
|
+
tenzir_test-0.11.0.dist-info/licenses/LICENSE,sha256=ajMbpcBiSTXI8Rr4t17pvowV-On8DktghfZKxY_A22Q,10750
|
|
29
|
+
tenzir_test-0.11.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|