tenzir-test 0.12.0__py3-none-any.whl

tenzir_test/runners/__init__.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Callable
+
+from .runner import Runner
+from .ext_runner import ExtRunner
+from .tql_runner import TqlRunner
+from .diff_runner import DiffRunner
+from .shell_runner import ShellRunner
+from .custom_python_fixture_runner import CustomPythonFixture
+from .tenzir_runner import TenzirRunner
+
+_REGISTERED: dict[str, Runner] = {}
+_ALIASES: dict[str, str] = {}
+RUNNERS: list[Runner] = []
+RUNNERS_BY_NAME: dict[str, Runner] = {}
+
+
+def _refresh_exports() -> None:
+    global RUNNERS, RUNNERS_BY_NAME
+    RUNNERS = list(_REGISTERED.values())
+    mapping: dict[str, Runner] = dict(_REGISTERED)
+    for alias, target in _ALIASES.items():
+        runner = mapping.get(target)
+        if runner is None:
+            continue
+        mapping[alias] = runner
+    RUNNERS_BY_NAME = mapping
+
+
+def _set_alias(alias: str, target: str, *, replace: bool) -> None:
+    if target not in _REGISTERED:
+        raise ValueError(f"runner '{target}' is not registered")
+    if alias in _REGISTERED and alias != target:
+        raise ValueError(f"cannot alias '{alias}': name already registered")
+    if not replace and alias in _ALIASES and _ALIASES[alias] != target:
+        raise ValueError(f"alias '{alias}' already targets '{_ALIASES[alias]}'")
+    _ALIASES[alias] = target
+
+
+def register(
+    runner: Runner,
+    *,
+    replace: bool = False,
+    aliases: Sequence[str] | None = None,
+) -> Runner:
+    name = runner.name
+    if not replace and name in _REGISTERED and _REGISTERED[name] is not runner:
+        raise ValueError(f"runner '{name}' already registered")
+    _REGISTERED[name] = runner
+    if aliases:
+        for alias in aliases:
+            _set_alias(alias, name, replace=replace)
+    _refresh_exports()
+    return runner
+
+
+def unregister(name: str) -> Runner:
+    try:
+        runner = _REGISTERED.pop(name)
+    except KeyError as exc:  # pragma: no cover - defensive guard
+        raise ValueError(f"runner '{name}' is not registered") from exc
+    to_remove = [alias for alias, target in _ALIASES.items() if target == name]
+    for alias in to_remove:
+        _ALIASES.pop(alias, None)
+    _refresh_exports()
+    return runner
+
+
+def register_alias(alias: str, target: str, *, replace: bool = False) -> None:
+    _set_alias(alias, target, replace=replace)
+    _refresh_exports()
+
+
+RunnerFactory = Callable[[], Runner]
+
+
+def startup(
+    *,
+    replace: bool = False,
+    aliases: Sequence[str] | None = None,
+) -> Callable[[RunnerFactory], RunnerFactory]:
+    def _decorator(factory: RunnerFactory) -> RunnerFactory:
+        runner = factory()
+        if not isinstance(runner, Runner):
+            raise TypeError("runner factory must return a Runner instance")
+        register(runner, replace=replace, aliases=aliases)
+        return factory
+
+    return _decorator
+
+
+def iter_runners() -> tuple[Runner, ...]:
+    return tuple(RUNNERS)
+
+
+def runner_map(*, include_aliases: bool = True) -> dict[str, Runner]:
+    if include_aliases:
+        return dict(RUNNERS_BY_NAME)
+    return dict(_REGISTERED)
+
+
+def runner_names() -> set[str]:
+    return set(RUNNERS_BY_NAME.keys())
+
+
+def allowed_extensions() -> set[str]:
+    extensions: set[str] = set()
+    for runner in RUNNERS:
+        ext = getattr(runner, "_ext", None)
+        if isinstance(ext, str):
+            extensions.add(ext)
+    return extensions
+
+
+def get_runner_for_test(test_path: Path) -> Runner:
+    from tenzir_test import run as run_module
+
+    try:
+        config = run_module.parse_test_config(test_path)
+    except ValueError:
+        suffix = test_path.suffix.lower()
+        default_name = run_module.default_runner_for_suffix(suffix)
+        if not default_name:
+            raise
+        runner_name = default_name
+    else:
+        runner_value = config.get("runner")
+        if not isinstance(runner_value, str):
+            raise ValueError("Runner 'runner' must be a string")
+        runner_name = runner_value
+    if runner_name in RUNNERS_BY_NAME:
+        return RUNNERS_BY_NAME[runner_name]
+    raise ValueError(f"Runner '{runner_name}' not found - this is a bug")
+
+
+register(ShellRunner())
+register(CustomPythonFixture())
+register(TenzirRunner())
+
+_refresh_exports()
+
+__all__ = [
+    "Runner",
+    "ExtRunner",
+    "TqlRunner",
+    "ShellRunner",
+    "CustomPythonFixture",
+    "TenzirRunner",
+    "DiffRunner",
+    "RUNNERS",
+    "RUNNERS_BY_NAME",
+    "register",
+    "unregister",
+    "register_alias",
+    "startup",
+    "iter_runners",
+    "runner_map",
+    "runner_names",
+    "allowed_extensions",
+    "get_runner_for_test",
+]

tenzir_test/runners/_utils.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import os
+from types import ModuleType
+
+
+def get_run_module() -> ModuleType:
+    from tenzir_test import run as run_module
+
+    return run_module
+
+
+def resolve_run_module_dir(run_mod: ModuleType) -> str:
+    module_path = getattr(run_mod, "__file__", None)
+    if not isinstance(module_path, str):
+        raise RuntimeError("tenzir_test.run module path is not available")
+    return os.path.dirname(os.path.realpath(module_path))

tenzir_test/runners/custom_python_fixture_runner.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import typing
+from pathlib import Path
+
+from tenzir_test import fixtures as fixture_api
+
+from ._utils import get_run_module, resolve_run_module_dir
+from .ext_runner import ExtRunner
+
+
+def _jsonify_config(config: dict[str, typing.Any]) -> dict[str, typing.Any]:
+    def _convert(value: typing.Any) -> typing.Any:
+        if isinstance(value, tuple):
+            return [_convert(item) for item in value]
+        if isinstance(value, list):
+            return [_convert(item) for item in value]
+        if isinstance(value, dict):
+            return {str(k): _convert(v) for k, v in value.items()}
+        return value
+
+    return {str(key): _convert(val) for key, val in config.items()}
+
+
+class CustomPythonFixture(ExtRunner):
+    def __init__(self) -> None:
+        super().__init__(name="python", ext="py")
+
+    def run(self, test: Path, update: bool, coverage: bool = False) -> bool:
+        run_mod = get_run_module()
+        test_config = run_mod.parse_test_config(test, coverage=coverage)
+        binary = run_mod.TENZIR_BINARY
+        if not binary:
+            raise RuntimeError("TENZIR_BINARY must be configured for python fixtures")
+        passthrough = run_mod.is_passthrough_enabled()
+        try:
+            cmd = [
+                sys.executable,
+                "-m",
+                "tenzir_test._python_runner",
+                str(test),
+            ]
+            inputs_override = typing.cast(str | None, test_config.get("inputs"))
+            env, _config_args = run_mod.get_test_env_and_config_args(test, inputs=inputs_override)
+            fixtures = typing.cast(tuple[str, ...], test_config.get("fixtures", tuple()))
+            node_requested = "node" in fixtures
+            timeout = typing.cast(int, test_config["timeout"])
+            fixture_context = fixture_api.FixtureContext(
+                test=test,
+                config=typing.cast(dict[str, typing.Any], test_config),
+                coverage=coverage,
+                env=env,
+                config_args=tuple(),
+                tenzir_binary=run_mod.TENZIR_BINARY,
+                tenzir_node_binary=run_mod.TENZIR_NODE_BINARY,
+            )
+            context_token = fixture_api.push_context(fixture_context)
+            pythonpath_entries: list[str] = []
+            project_root = getattr(run_mod, "ROOT", None)
+            project_root_path: Path | None = None
+            if isinstance(project_root, Path):
+                project_root_path = project_root
+            elif isinstance(project_root, str):
+                project_root_path = Path(project_root)
+
+            if project_root_path is not None:
+                pythonpath_entries.append(str(project_root_path))
+                tests_dir = project_root_path / "tests"
+                if tests_dir.is_dir():
+                    pythonpath_entries.append(str(tests_dir))
+            pythonpath_entries.append(resolve_run_module_dir(run_mod))
+            existing_pythonpath = env.get("PYTHONPATH")
+            if existing_pythonpath:
+                pythonpath_entries.append(existing_pythonpath)
+
+            env["TENZIR_PYTHON_FIXTURE_CONTEXT"] = json.dumps(
+                {
+                    "test": str(test),
+                    "config": _jsonify_config(test_config),
+                    "coverage": coverage,
+                    "config_args": list(fixture_context.config_args),
+                }
+            )
+            try:
+                with fixture_api.activate(fixtures) as fixture_env:
+                    env.update(fixture_env)
+                    run_mod._apply_fixture_env(env, fixtures)
+                    if node_requested:
+                        endpoint = env.get("TENZIR_NODE_CLIENT_ENDPOINT")
+                        if not endpoint:
+                            raise RuntimeError(
+                                "node fixture did not provide TENZIR_NODE_CLIENT_ENDPOINT"
+                            )
+                    else:
+                        endpoint = None
+                    new_pythonpath = os.pathsep.join(pythonpath_entries)
+                    env["PYTHONPATH"] = new_pythonpath
+                    env["TENZIR_NODE_CLIENT_BINARY"] = binary
+                    env["TENZIR_NODE_CLIENT_TIMEOUT"] = str(timeout)
+                    env.setdefault("TENZIR_PYTHON_FIXTURE_BINARY", binary)
+                    env["TENZIR_PYTHON_FIXTURE_TIMEOUT"] = str(timeout)
+                    if node_requested and endpoint:
+                        env["TENZIR_PYTHON_FIXTURE_ENDPOINT"] = endpoint
+                    completed = run_mod.run_subprocess(
+                        cmd,
+                        timeout=timeout,
+                        env=env,
+                        capture_output=not passthrough,
+                        check=True,
+                    )
+                ref_path = test.with_suffix(".txt")
+                if completed.returncode != 0:
+                    run_mod.fail(test)
+                    return False
+                if passthrough:
+                    run_mod.success(test)
+                    return True
+                output = (completed.stdout or b"") + (completed.stderr or b"")
+                if update:
+                    with open(ref_path, "wb") as f:
+                        f.write(output)
+                else:
+                    if not ref_path.exists():
+                        run_mod.report_failure(
+                            test,
+                            run_mod.format_failure_message(
+                                f'Failed to find ref file: "{ref_path}"'
+                            ),
+                        )
+                        return False
+                    run_mod.log_comparison(test, ref_path, mode="comparing")
+                    expected = ref_path.read_bytes()
+                    if expected != output:
+                        if run_mod.interrupt_requested():
+                            run_mod.report_interrupted_test(test)
+                        else:
+                            run_mod.report_failure(test, "")
+                            run_mod.print_diff(expected, output, ref_path)
+                        return False
+            finally:
+                fixture_api.pop_context(context_token)
+                run_mod.cleanup_test_tmp_dir(env.get(run_mod.TEST_TMP_ENV_VAR))
+        except subprocess.TimeoutExpired:
+            run_mod.report_failure(
+                test,
+                run_mod.format_failure_message(f"python fixture hit {timeout}s timeout"),
+            )
+            return False
+        except subprocess.CalledProcessError as e:
+            suppressed = run_mod.should_suppress_failure_output()
+            if suppressed:
+                return False
+            with run_mod.stdout_lock:
+                run_mod.fail(test)
+                if not passthrough:
+                    if e.stdout:
+                        sys.stdout.buffer.write(e.stdout)
+                    if e.output and e.output is not e.stdout:
+                        sys.stdout.buffer.write(e.output)
+                    if e.stderr:
+                        sys.stdout.buffer.write(e.stderr)
+            return False
+        run_mod.success(test)
+        return True
+
+
+__all__ = ["CustomPythonFixture"]

tenzir_test/runners/diff_runner.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import difflib
+import os
+import typing
+from pathlib import Path
+
+from tenzir_test import fixtures as fixture_api
+
+from ._utils import get_run_module
+from .tql_runner import TqlRunner
+
+
+class DiffRunner(TqlRunner):
+    def __init__(self, *, a: str, b: str, name: str) -> None:
+        super().__init__(name=name)
+        self._a = a
+        self._b = b
+
+    def run(self, test: Path, update: bool, coverage: bool = False) -> bool | str:
+        run_mod = get_run_module()
+        test_config = run_mod.parse_test_config(test, coverage=coverage)
+        skip_value = test_config.get("skip")
+        if isinstance(skip_value, str):
+            return typing.cast(
+                bool | str,
+                run_mod.handle_skip(
+                    skip_value,
+                    test,
+                    update=update,
+                    output_ext=self.output_ext,
+                ),
+            )
+
+        inputs_override = typing.cast(str | None, test_config.get("inputs"))
+        env, config_args = run_mod.get_test_env_and_config_args(test, inputs=inputs_override)
+        fixtures = typing.cast(tuple[str, ...], test_config.get("fixtures", tuple()))
+        node_requested = "node" in fixtures
+        timeout = typing.cast(int, test_config["timeout"])
+
+        context_token = fixture_api.push_context(
+            fixture_api.FixtureContext(
+                test=test,
+                config=typing.cast(dict[str, typing.Any], test_config),
+                coverage=coverage,
+                env=env,
+                config_args=tuple(config_args),
+                tenzir_binary=run_mod.TENZIR_BINARY,
+                tenzir_node_binary=run_mod.TENZIR_NODE_BINARY,
+            )
+        )
+        try:
+            with fixture_api.activate(fixtures) as fixture_env:
+                env.update(fixture_env)
+                run_mod._apply_fixture_env(env, fixtures)
+
+                node_args: list[str] = []
+                if node_requested:
+                    endpoint = env.get("TENZIR_NODE_CLIENT_ENDPOINT")
+                    if not endpoint:
+                        raise RuntimeError(
+                            "node fixture did not provide TENZIR_NODE_CLIENT_ENDPOINT"
+                        )
+                    node_args.append(f"--endpoint={endpoint}")
+
+                binary = run_mod.TENZIR_BINARY
+                if not binary:
+                    raise RuntimeError("TENZIR_BINARY must be configured for diff runners")
+                base_cmd: list[str] = [binary, *config_args]
+
+                if coverage:
+                    coverage_dir = env.get(
+                        "CMAKE_COVERAGE_OUTPUT_DIRECTORY",
+                        os.path.join(os.getcwd(), "coverage"),
+                    )
+                    source_dir = env.get("COVERAGE_SOURCE_DIR", os.getcwd())
+                    os.makedirs(coverage_dir, exist_ok=True)
+                    env["COVERAGE_SOURCE_DIR"] = source_dir
+                    env["LLVM_PROFILE_FILE"] = os.path.join(
+                        coverage_dir, f"{test.stem}-unopt-%p.profraw"
+                    )
+
+                unoptimized = run_mod.run_subprocess(
+                    [*base_cmd, self._a, *node_args, "-f", str(test)],
+                    timeout=timeout,
+                    env=env,
+                    capture_output=True,
+                    check=False,
+                    force_capture=True,
+                )
+
+                if coverage:
+                    env["LLVM_PROFILE_FILE"] = os.path.join(
+                        coverage_dir, f"{test.stem}-opt-%p.profraw"
+                    )
+
+                optimized = run_mod.run_subprocess(
+                    [*base_cmd, self._b, *node_args, "-f", str(test)],
+                    timeout=timeout,
+                    env=env,
+                    capture_output=True,
+                    check=False,
+                    force_capture=True,
+                )
+        finally:
+            fixture_api.pop_context(context_token)
+            run_mod.cleanup_test_tmp_dir(env.get(run_mod.TEST_TMP_ENV_VAR))
+
+        diff_chunks = list(
+            difflib.diff_bytes(
+                difflib.unified_diff,
+                unoptimized.stdout.splitlines(keepends=True),
+                optimized.stdout.splitlines(keepends=True),
+                n=2**31 - 1,
+            )
+        )[3:]
+        if diff_chunks:
+            diff_bytes = b"".join(diff_chunks)
+        else:
+            diff_bytes = b"".join(
+                b" " + line for line in unoptimized.stdout.splitlines(keepends=True)
+            )
+        ref_path = test.with_suffix(".diff")
+        if update:
+            ref_path.write_bytes(diff_bytes)
+        else:
+            expected = ref_path.read_bytes()
+            if diff_bytes != expected:
+                if run_mod.interrupt_requested():
+                    run_mod.report_interrupted_test(test)
+                else:
+                    run_mod.report_failure(test, "")
+                    run_mod.print_diff(expected, diff_bytes, ref_path)
+                return False
+        run_mod.success(test)
+        return True
+
+
+__all__ = ["DiffRunner"]

tenzir_test/runners/ext_runner.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ._utils import get_run_module
+from .runner import Runner
+
+
+class ExtRunner(Runner):
+    def __init__(self, *, name: str, ext: str) -> None:
+        super().__init__(name=name)
+        self._ext = ext
+
+    def collect_tests(self, path: Path) -> set[tuple[Runner, Path]]:
+        return self.collect_with_ext(path, self._ext)
+
+    def purge(self) -> None:
+        run_mod = get_run_module()
+        purge_base = run_mod.ROOT / self._name
+        if not purge_base.exists():
+            return
+        for p in purge_base.rglob("*"):
+            if p.is_dir() or p.suffix == f".{self._ext}":
+                continue
+            p.unlink()
+
+
+__all__ = ["ExtRunner"]

tenzir_test/runners/runner.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class Runner(ABC):
+    def __init__(self, *, name: str) -> None:
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def collect_with_ext(self, path: Path, ext: str) -> set[tuple[Runner, Path]]:
+        todo: set[tuple[Runner, Path]] = set()
+        if path.is_file():
+            if path.suffix == f".{ext}":
+                todo.add((self, path))
+            return todo
+        for test in path.glob(f"**/*.{ext}"):
+            todo.add((self, test))
+        return todo
+
+    @abstractmethod
+    def collect_tests(self, path: Path) -> set[tuple[Runner, Path]]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def purge(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def run(self, test: Path, update: bool, coverage: bool = False) -> bool | str:
+        raise NotImplementedError
+
+
+__all__ = ["Runner"]