tenzir-test 0.9.4__tar.gz → 0.10.0__tar.gz

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tenzir-test
-Version: 0.9.4
+Version: 0.10.0
 Summary: Reusable test execution framework extracted from the Tenzir repository.
 Project-URL: Homepage, https://github.com/tenzir/test
 Project-URL: Repository, https://github.com/tenzir/test
@@ -55,36 +55,6 @@ uvx tenzir-test --help
 `uvx` downloads the newest compatible release, runs it in an isolated
 environment, and caches subsequent invocations for fast reuse.
 
-## 🚀 Quick Start
-
-Create a project skeleton that mirrors the layout the harness expects:
-
-```text
-project-root/
-├── fixtures/
-│   └── http.py
-├── inputs/
-│   └── sample.ndjson
-├── runners/
-│   └── __init__.py
-└── tests/
-    ├── alerts/
-    │   ├── sample.py
-    │   └── sample.txt
-    └── regression/
-        ├── dummy.tql
-        └── dummy.txt
-```
-
-1. Author fixtures in `fixtures/` and register them at import time.
-2. Store reusable datasets in `inputs/`—the harness exposes the path via
-   `TENZIR_INPUTS` and provides a per-test scratch directory through
-   `TENZIR_TMP_DIR` when tests execute.
-   Use `--keep` (or `-k`) to preserve those temporary directories for debugging.
-3. Create tests in `tests/` and pair them with reference artifacts (for example
-   `.txt`) that the harness compares against.
-4. Run `uvx tenzir-test` from the project root to execute the full suite.
-
 ## 📚 Documentation
 
 Consult our [user guide](https://docs.tenzir.com/guides/testing/write-tests)

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/README.md

@@ -26,36 +26,6 @@ uvx tenzir-test --help
 `uvx` downloads the newest compatible release, runs it in an isolated
 environment, and caches subsequent invocations for fast reuse.
 
-## 🚀 Quick Start
-
-Create a project skeleton that mirrors the layout the harness expects:
-
-```text
-project-root/
-├── fixtures/
-│   └── http.py
-├── inputs/
-│   └── sample.ndjson
-├── runners/
-│   └── __init__.py
-└── tests/
-    ├── alerts/
-    │   ├── sample.py
-    │   └── sample.txt
-    └── regression/
-        ├── dummy.tql
-        └── dummy.txt
-```
-
-1. Author fixtures in `fixtures/` and register them at import time.
-2. Store reusable datasets in `inputs/`—the harness exposes the path via
-   `TENZIR_INPUTS` and provides a per-test scratch directory through
-   `TENZIR_TMP_DIR` when tests execute.
-   Use `--keep` (or `-k`) to preserve those temporary directories for debugging.
-3. Create tests in `tests/` and pair them with reference artifacts (for example
-   `.txt`) that the harness compares against.
-4. Run `uvx tenzir-test` from the project root to execute the full suite.
-
 ## 📚 Documentation
 
 Consult our [user guide](https://docs.tenzir.com/guides/testing/write-tests)

tenzir_test-0.10.0/example-project/tests/shell/exit-code-test.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# This test intentionally fails to demonstrate the arrow output
+
+echo "Running test..."
+echo "About to fail with exit code 42"
+echo "Simulated error message on stderr" >&2
+exit 42

tenzir_test-0.10.0/example-project/tests/shell/exit-code-test.txt

@@ -0,0 +1,2 @@
+Running test...
+About to fail with exit code 42

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tenzir-test"
-version = "0.9.4"
+version = "0.10.0"
 description = "Reusable test execution framework extracted from the Tenzir repository."
 readme = "README.md"
 requires-python = ">=3.12"

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/src/tenzir_test/run.py

@@ -169,6 +169,7 @@ PASS_MAX_COLOR = "\033[92m"
 FAIL_COLOR = "\033[31m"
 SKIP_COLOR = "\033[90;1m"
 RESET_COLOR = "\033[0m"
+DETAIL_COLOR = "\033[2;37m"
 PASS_SPECTRUM = [
     "\033[38;5;52m",  # 0-9%   deep red
     "\033[38;5;88m",  # 10-19% red
@@ -349,6 +350,7 @@ _TMP_ROOT_NAME = ".tenzir-test"
 _TMP_SUBDIR_NAME = "tmp"
 _TMP_BASE_DIRS: set[Path] = set()
 _ACTIVE_TMP_DIRS: set[Path] = set()
+_TMP_DIR_LOCK = threading.Lock()
 KEEP_TMP_DIRS = bool(os.environ.get(_TMP_KEEP_ENV_VAR))
 
 SHOW_DIFF_OUTPUT = True
@@ -506,10 +508,13 @@ def _tmp_prefix_for(test: Path) -> str:
 
 
 def _create_test_tmp_dir(test: Path) -> Path:
-    base = _resolve_tmp_base()
     prefix = f"{_tmp_prefix_for(test)}-"
-    path = Path(tempfile.mkdtemp(prefix=prefix, dir=str(base)))
-    _ACTIVE_TMP_DIRS.add(path)
+    with _TMP_DIR_LOCK:
+        base = _resolve_tmp_base()
+        if not base.exists():
+            base.mkdir(parents=True, exist_ok=True)
+        path = Path(tempfile.mkdtemp(prefix=prefix, dir=str(base)))
+        _ACTIVE_TMP_DIRS.add(path)
     return path
 
 
@@ -522,22 +527,27 @@ def cleanup_test_tmp_dir(path: str | os.PathLike[str] | None) -> None:
     if not path:
         return
     tmp_path = Path(path)
-    _ACTIVE_TMP_DIRS.discard(tmp_path)
+    with _TMP_DIR_LOCK:
+        _ACTIVE_TMP_DIRS.discard(tmp_path)
     try:
         fixtures_impl.invoke_tmp_dir_cleanup(tmp_path)
     except Exception:  # pragma: no cover - defensive logging
         pass
     if KEEP_TMP_DIRS:
         return
-    if tmp_path.exists():
-        shutil.rmtree(tmp_path, ignore_errors=True)
-    _cleanup_tmp_base_dirs()
+    with _TMP_DIR_LOCK:
+        if tmp_path.exists():
+            shutil.rmtree(tmp_path, ignore_errors=True)
+        _cleanup_tmp_base_dirs()
 
 
 def _cleanup_remaining_tmp_dirs() -> None:
-    for tmp_path in list(_ACTIVE_TMP_DIRS):
+    with _TMP_DIR_LOCK:
+        remaining = list(_ACTIVE_TMP_DIRS)
+    for tmp_path in remaining:
         cleanup_test_tmp_dir(tmp_path)
-    _cleanup_tmp_base_dirs()
+    with _TMP_DIR_LOCK:
+        _cleanup_tmp_base_dirs()
 
 
 def _cleanup_all_tmp_dirs() -> None:
@@ -545,9 +555,12 @@ def _cleanup_all_tmp_dirs() -> None:
 
     if KEEP_TMP_DIRS:
         return
-    for tmp_path in list(_ACTIVE_TMP_DIRS):
+    with _TMP_DIR_LOCK:
+        remaining = list(_ACTIVE_TMP_DIRS)
+    for tmp_path in remaining:
         cleanup_test_tmp_dir(tmp_path)
-    _cleanup_tmp_base_dirs()
+    with _TMP_DIR_LOCK:
+        _cleanup_tmp_base_dirs()
 
 
 def _cleanup_tmp_base_dirs() -> None:
@@ -2620,9 +2633,13 @@ def run_simple_test(
             )
         good = completed.returncode == 0
         output = b""
+        stderr_output = b""
         if not passthrough_mode:
-            captured = completed.stdout or b""
-            output = captured.replace(str(ROOT).encode() + b"/", b"")
+            root_bytes = str(ROOT).encode() + b"/"
+            captured_stdout = completed.stdout or b""
+            output = captured_stdout.replace(root_bytes, b"")
+            captured_stderr = completed.stderr or b""
+            stderr_output = captured_stderr.replace(root_bytes, b"")
     except subprocess.TimeoutExpired:
         report_failure(test, f"└─▶ \033[31msubprocess hit {timeout}s timeout\033[0m")
         return False
@@ -2642,20 +2659,30 @@ def run_simple_test(
             return False
         if interrupted:
             _request_interrupt()
-        message = (
+        summary_line = (
             _INTERRUPTED_NOTICE
             if interrupted
-            else f"┌─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m"
+            else f"└─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m"
         )
         if passthrough_mode:
-            report_failure(test, message)
+            report_failure(test, summary_line)
         else:
             with stdout_lock:
-                report_failure(test, message)
+                fail(test)
                 if not interrupted:
-                    for last, line in last_and(output.split(b"\n")):
-                        prefix = "│ " if not last else "└─"
-                        sys.stdout.buffer.write(prefix.encode() + line + b"\n")
+                    line_prefix = "│ ".encode()
+                    for line in output.splitlines():
+                        sys.stdout.buffer.write(line_prefix + line + b"\n")
+                    if completed.returncode != 0 and stderr_output:
+                        sys.stdout.write("├─▶ stderr\n")
+                        detail_prefix = DETAIL_COLOR.encode()
+                        reset_bytes = RESET_COLOR.encode()
+                        for line in stderr_output.splitlines():
+                            sys.stdout.buffer.write(
+                                line_prefix + detail_prefix + line + reset_bytes + b"\n"
+                            )
+                if summary_line:
+                    sys.stdout.write(summary_line + "\n")
         return False
     if passthrough_mode:
         success(test)

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/src/tenzir_test/runners/shell_runner.py

@@ -64,47 +64,49 @@ class ShellRunner(ExtRunner):
             fixture_api.pop_context(context_token)
             run_mod.cleanup_test_tmp_dir(env.get(run_mod.TEST_TMP_ENV_VAR))
 
+        stdout_data = completed.stdout
+        if isinstance(stdout_data, str):
+            stdout_bytes: bytes = stdout_data.encode()
+        else:
+            stdout_bytes = stdout_data or b""
+
+        stderr_data = completed.stderr
+        if isinstance(stderr_data, str):
+            stderr_bytes: bytes = stderr_data.encode()
+        else:
+            stderr_bytes = stderr_data or b""
+
         good = completed.returncode == 0
         if expect_error == good:
             suppressed = run_mod.should_suppress_failure_output()
+            summary_line = f"└─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m"
             if passthrough:
                 if not suppressed:
-                    run_mod.report_failure(
-                        test,
-                        f"┌─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m",
-                    )
+                    run_mod.report_failure(test, summary_line)
                 return False
             if suppressed:
                 return False
+
             with run_mod.stdout_lock:
-                run_mod.report_failure(
-                    test,
-                    f"┌─▶ \033[31mgot unexpected exit code {completed.returncode}\033[0m",
-                )
-                stdout = completed.stdout or b""
-                for last, line in run_mod.last_and(stdout.split(b"\n")):
-                    prefix = "│ " if not last else "└─"
-                    sys.stdout.buffer.write(prefix.encode() + line + b"\n")
-                if completed.stderr:
-                    sys.stdout.buffer.write(completed.stderr)
+                run_mod.fail(test)
+                line_prefix = "│ ".encode()
+                for line in stdout_bytes.splitlines():
+                    sys.stdout.buffer.write(line_prefix + line + b"\n")
+                if stderr_bytes:
+                    sys.stdout.write("├─▶ stderr\n")
+                    detail_prefix = run_mod.DETAIL_COLOR.encode()
+                    reset_bytes = run_mod.RESET_COLOR.encode()
+                    for line in stderr_bytes.splitlines():
+                        sys.stdout.buffer.write(
+                            line_prefix + detail_prefix + line + reset_bytes + b"\n"
+                        )
+                sys.stdout.write(summary_line + "\n")
             return False
 
         if passthrough:
             run_mod.success(test)
             return True
 
-        stdout_data = completed.stdout
-        if isinstance(stdout_data, str):
-            stdout_bytes: bytes = stdout_data.encode()
-        else:
-            stdout_bytes = stdout_data or b""
-
-        stderr_data = completed.stderr
-        if isinstance(stderr_data, str):
-            stderr_bytes: bytes = stderr_data.encode()
-        else:
-            stderr_bytes = stderr_data or b""
-
         root_prefix: bytes = (str(run_mod.ROOT) + "/").encode()
         stdout_bytes = stdout_bytes.replace(root_prefix, b"")
 

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/tests/test_run_config.py

@@ -502,6 +502,7 @@ def test_run_simple_test_injects_package_dirs(
         def __init__(self) -> None:
             self.returncode = 0
             self.stdout = b"ok"
+            self.stderr = b""
 
     def fake_run(cmd, timeout, stdout=None, stderr=None, env=None, **kwargs):  # type: ignore[no-untyped-def]
         captured["cmd"] = list(cmd)
@@ -563,6 +564,7 @@ def test_run_simple_test_respects_inputs_override(
         def __init__(self) -> None:
             self.returncode = 0
             self.stdout = b"ok"
+            self.stderr = b""
 
     def fake_run(cmd, timeout, stdout=None, stderr=None, env=None, **kwargs):  # type: ignore[no-untyped-def]
         captured["cmd"] = list(cmd)
@@ -634,6 +636,48 @@ def test_run_simple_test_passthrough_streams_output(
     assert captured.get("stdout") is None
 
 
+def test_run_simple_test_reports_stderr_on_failure(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    test_file = tmp_path / "case.tql"
+    test_file.write_text("version\nwrite_json\n", encoding="utf-8")
+
+    original_settings = config.Settings(
+        root=run.ROOT,
+        tenzir_binary=run.TENZIR_BINARY,
+        tenzir_node_binary=run.TENZIR_NODE_BINARY,
+    )
+    run.apply_settings(
+        config.Settings(
+            root=tmp_path,
+            tenzir_binary=sys.executable,
+            tenzir_node_binary=None,
+        )
+    )
+
+    class FakeCompletedProcess:
+        def __init__(self) -> None:
+            self.returncode = 13
+            self.stdout = b"captured stdout\n"
+            self.stderr = b"captured stderr\n"
+
+    monkeypatch.setattr(run, "run_subprocess", lambda *args, **kwargs: FakeCompletedProcess())
+
+    try:
+        result = run.run_simple_test(test_file, update=False, output_ext="txt")
+    finally:
+        run.apply_settings(original_settings)
+
+    assert result is False
+    lines = capsys.readouterr().out.splitlines()
+    assert "✘" in lines[0] and lines[0].endswith("case.tql")
+    assert run.ANSI_ESCAPE.sub("", lines[1]) == "│ captured stdout"
+    assert run.ANSI_ESCAPE.sub("", lines[2]) == "├─▶ stderr"
+    assert run.ANSI_ESCAPE.sub("", lines[3]) == "│ captured stderr"
+    assert lines[4].startswith("└─▶ ")
+    assert "got unexpected exit code 13" in lines[4]
+
+
 def test_run_simple_test_suppresses_diff_on_interrupt(
     monkeypatch: pytest.MonkeyPatch, tmp_path: Path
 ) -> None:

{tenzir_test-0.9.4 → tenzir_test-0.10.0}/tests/test_shell_runner.py

@@ -145,3 +145,48 @@ def test_shell_runner_passthrough_streams_output(
     assert captured["check"] is True
     assert captured["capture_output"] is False
     assert captured["cwd"] == str(tmp_path)
+
+
+def test_shell_runner_reports_stderr_on_failure(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    script = tmp_path / "tests" / "shell" / "fail.sh"
+    script.parent.mkdir(parents=True, exist_ok=True)
+    script.write_text("echo stdout\n >&2 echo stderr\n exit 42\n", encoding="utf-8")
+    script.chmod(0o755)
+
+    original_settings = config.Settings(
+        root=run.ROOT,
+        tenzir_binary=run.TENZIR_BINARY,
+        tenzir_node_binary=run.TENZIR_NODE_BINARY,
+    )
+
+    class FakeCompletedProcess:
+        def __init__(self) -> None:
+            self.returncode = 42
+            self.stdout = b"stdout\n"
+            self.stderr = b"stderr\n"
+
+    monkeypatch.setattr(run, "run_subprocess", lambda *args, **kwargs: FakeCompletedProcess())
+
+    try:
+        run.apply_settings(
+            config.Settings(
+                root=tmp_path,
+                tenzir_binary=run.TENZIR_BINARY,
+                tenzir_node_binary=run.TENZIR_NODE_BINARY,
+            )
+        )
+        runner = run.ShellRunner()
+        result = runner.run(script, update=False, coverage=False)
+    finally:
+        run.apply_settings(original_settings)
+
+    assert result is False
+    lines = capsys.readouterr().out.splitlines()
+    assert "✘" in lines[0] and lines[0].endswith("tests/shell/fail.sh")
+    assert run.ANSI_ESCAPE.sub("", lines[1]) == "│ stdout"
+    assert run.ANSI_ESCAPE.sub("", lines[2]) == "├─▶ stderr"
+    assert run.ANSI_ESCAPE.sub("", lines[3]) == "│ stderr"
+    assert lines[4].startswith("└─▶ ")
+    assert "got unexpected exit code 42" in lines[4]