testaferro 0.1.0.dev0__py3-none-any.whl

testaferro/__init__.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""testaferro: a pytest facade for remote (e.g. QEMU-guest) unit
+tests.
+
+A suite compiled for and running on a remote target surfaces as
+ordinary pytest tests on the host. Hand guest_suite() a reference to
+the suite executable:
+
+    from pathlib import Path
+
+    import testaferro
+
+    test_guest_case = testaferro.guest_suite(
+        Path(__file__).parent / "SUITE.EXE")
+
+The executable is interrogated to select its platform (DOS programs
+run in a QEMU guest; anything else is rejected), the
+framework adapter defaults to testaferro.cpputest (`framework=`
+overrides), and the runner's working state lives in
+testaferro-managed disposable directories. Named test machines are
+declared with config() or an optional per-project testaferro.ini; a
+prebuilt Backend remains the custom escape hatch for callers that
+need a different execution mechanism.
+
+For many suites (and future parallel runs), open a session so the
+boot image is specified once and all per-run state is swept together
+— in pytest, from the consumer's conftest.py:
+
+    import testaferro
+
+    testaferro.start()              # or start(boot_image=...)
+
+    def pytest_unconfigure(config):
+        testaferro.stop()
+"""
+
+# eager: the facade's own imports are stdlib-only (pytest is loaded
+# lazily inside it). start/stop delegate lazily instead, because
+# importing testaferro.qemu pulls in reliquary.
+from .facade import guest_suite  # noqa: F401
+
+
+def config(machine, platform=None, **options):
+    """Declare a named reliquary test machine.
+
+    ``platform`` is optional when the supplied ``machine_config`` or
+    ``template`` declares it. Without a template, remaining options are
+    the blueprint's own machine fields. The declaration is reused
+    as a template; each guest session receives a fresh materialization.
+    The same declarations may be written in ``testaferro.ini`` (see
+    ``load_config``).
+    """
+    from .machines import configure
+    return configure(machine, platform=platform, **options)
+
+
+def load_config(path=None):
+    """Load named machines from a ``testaferro.ini``.
+
+    With ``path``, read that file. With ``path`` omitted, search
+    upward from the current directory. ``guest_suite()`` performs the
+    same search from its call site automatically.
+    """
+    from .machines import load_config as load
+    return load(path)
+
+
+def start(boot_image=None):
+    """Open a guest-test session: one boot-image choice (or the
+    downloaded default) serving every suite until stop(). Costs
+    nothing until a guest actually runs; an atexit failsafe sweeps
+    the session if stop() is never called."""
+    from . import qemu
+    qemu.start(boot_image=boot_image)
+
+
+def stop(clear_downloads=False):
+    """Close the session, sweeping its staged image and all per-run
+    state; safe without an active session. `clear_downloads=True`
+    also drops the cached default boot image."""
+    from . import qemu
+    qemu.stop(clear_downloads=clear_downloads)

testaferro/backend.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""The Backend seam: what the pytest facade needs from any remote
+test target.
+
+Any target that can list its tests and run one or all of them plugs
+into the same facade, however it carries those operations out
+underneath. Session hooks default to no-ops because one-shot backends
+(e.g. a SuiteBackend whose runner boots per operation)
+have no session to manage.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TestId:
+    __test__ = False        # not a test case, despite the name
+
+    group: str
+    name: str
+
+    def __str__(self):
+        return f"{self.group}.{self.name}"
+
+
+@dataclass
+class TestOutcome:
+    __test__ = False        # not a test case, despite the name
+
+    group: str
+    name: str
+    passed: bool
+    file: str = ""
+    line: int = 0
+    message: str = ""
+
+
+class Backend(ABC):
+    def start_session(self):
+        """Boot or attach to the remote target. Ready to accept
+        list/run calls immediately after this returns."""
+
+    @abstractmethod
+    def list_tests(self) -> "list[TestId]":
+        ...
+
+    @abstractmethod
+    def run_test(self, group, name) -> TestOutcome:
+        """Run one test. Raises LookupError if the target did not
+        run it (e.g. host and target test lists diverged)."""
+
+    @abstractmethod
+    def run_all(self) -> "list[TestOutcome]":
+        """Used by the facade's batching logic when no host-side
+        filter is active - a single bulk operation is expected to be
+        cheaper than many individual run_test() calls, however the
+        backend chooses to implement that."""
+
+    def stop_session(self):
+        """Tear down cleanly. Must be safe to call after a failed or
+        partial start_session()."""

testaferro/binfmt.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""Executable-format classification for platform dispatch.
+
+`classify()` reads an executable's header and reports which platform
+could run it, without importing any runner:
+
+- ``Format("dos", ...)`` — a plain MZ program, or a headerless image
+  (`.com`-style raw code carries nothing to prove, so it passes
+  through for the guest itself to judge);
+- ``Format(None, ...)`` — a provably unsupported binary: PE,
+  NE/LX/LE, ELF, Mach-O (including universal).
+
+`kind` always carries the human-readable format-and-architecture
+name ('a Windows x64 (PE)', 'an ELF x86-64 (Linux/BSD)', ...) for
+error messages, and a future platform extends the classification by
+claiming formats currently mapped to None. Stdlib-only: the facade's
+dispatch and each platform binding's own guard share it without
+pulling in a runner.
+"""
+
+from __future__ import annotations
+
+import collections
+
+Format = collections.namedtuple("Format", ["platform", "kind"])
+
+_DOS_MZ = Format("dos", "a DOS MZ")
+_HEADERLESS = Format("dos", "a headerless (.com-style)")
+
+# architecture names per format, keyed by each format's machine field
+_ELF_MACHINES = {0x03: "x86", 0x28: "ARM", 0x3E: "x86-64",
+                 0xB7: "ARM64", 0xF3: "RISC-V"}
+_PE_MACHINES = {0x014C: "x86", 0x01C0: "ARM", 0x01C4: "ARM",
+                0x8664: "x64", 0xAA64: "ARM64"}
+_MACHO_CPUTYPES = {7: "x86", 0x01000007: "x86-64",
+                   12: "ARM", 0x0100000C: "ARM64"}
+
+
+def _elf_format(header):
+    """Kind string for an ELF image (Linux and the BSDs; most carry
+    the generic System V OS/ABI byte, so no OS is claimed)."""
+    arch = None
+    if len(header) >= 0x14:
+        order = "little" if header[5] == 1 else "big"
+        arch = _ELF_MACHINES.get(
+            int.from_bytes(header[0x12:0x14], order))
+    return (f"an ELF {arch} (Linux/BSD)" if arch
+            else "an ELF (Linux/BSD)")
+
+
+def _macho_format(header):
+    """Kind string for a Mach-O image, or None when the magic is
+    really something else's."""
+    magic = header[:4]
+    if magic in (b"\xca\xfe\xba\xbe", b"\xca\xfe\xba\xbf"):
+        # a universal binary's big-endian arch count is tiny; a Java
+        # class file shares the magic but puts its version there
+        if (len(header) >= 8
+                and int.from_bytes(header[4:8], "big") < 0x40):
+            return "a macOS universal (Mach-O)"
+        return None
+    order = "big" if magic[:2] == b"\xfe\xed" else "little"
+    arch = (_MACHO_CPUTYPES.get(int.from_bytes(header[4:8], order))
+            if len(header) >= 8 else None)
+    return (f"a macOS {arch} (Mach-O)" if arch
+            else "a macOS (Mach-O)")
+
+
+def _mz_extension_format(found):
+    """The Format of the newer-format header an extended MZ file
+    points at through e_lfanew, or None when none is recognized
+    there. DOS runs none of these."""
+    if found.startswith(b"PE\0\0"):
+        arch = (_PE_MACHINES.get(int.from_bytes(found[4:6], "little"))
+                if len(found) >= 6 else None)
+        return Format(None, f"a Windows {arch} (PE)" if arch
+                      else "a Windows (PE)")
+    for signature, kind in ((b"NE", "a 16-bit Windows or OS/2 (NE)"),
+                            (b"LX", "an OS/2 (LX)"),
+                            (b"LE", "a linear (LE)")):
+        if found.startswith(signature):
+            return Format(None, kind)
+    return None
+
+
+def classify(exe_path):
+    """The Format of the referenced executable: which platform could
+    run it (`platform`) and its human-readable kind. Raises
+    FileNotFoundError for a missing file; attaches no meaning beyond
+    the header — a "dos" verdict means "nothing proves otherwise"."""
+    with open(exe_path, "rb") as f:
+        header = f.read(0x40)
+        if header[:4] == b"\x7fELF":
+            return Format(None, _elf_format(header))
+        if header[:4] in (b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf",
+                          b"\xce\xfa\xed\xfe", b"\xcf\xfa\xed\xfe",
+                          b"\xca\xfe\xba\xbe", b"\xca\xfe\xba\xbf"):
+            kind = _macho_format(header)
+            return _HEADERLESS if kind is None else Format(None, kind)
+        if header[:2] != b"MZ":
+            return _HEADERLESS
+        if len(header) < 0x40:
+            return _DOS_MZ
+        if int.from_bytes(header[0x18:0x1A], "little") < 0x40:
+            return _DOS_MZ
+        f.seek(int.from_bytes(header[0x3C:0x40], "little"))
+        found = f.read(8)
+    return _mz_extension_format(found) or _DOS_MZ

testaferro/cache.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""testaferro's durable filespace, shared by the guest bindings."""
+
+from __future__ import annotations
+
+import os
+
+
+def cache_root():
+    """testaferro's own filespace: each guest binding's image cache
+    plus its disposable per-session runner homes (under runs/; stale
+    ones from killed processes can be deleted freely)."""
+    if os.name == "nt":
+        base = (os.environ.get("LOCALAPPDATA")
+                or os.path.join(os.path.expanduser("~"),
+                                "AppData", "Local"))
+    else:
+        base = (os.environ.get("XDG_CACHE_HOME")
+                or os.path.join(os.path.expanduser("~"), ".cache"))
+    return os.path.join(base, "testaferro")

testaferro/cpputest.py

@@ -0,0 +1,128 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""CppUTest framework adapter.
+
+Knows CppUTest and nothing else: the argv that makes a suite
+executable enumerate or run tests, and the grammar of the resulting
+output. How and where the executable runs — the guest-OS aspect — is
+deliberately not this module's business; the
+two aspects compose in a SuiteBackend (testaferro.suite) or directly
+at the call site:
+
+    log = run_guest_program(exe, args=cpputest.VERBOSE_ARGS)
+    results = cpputest.parse(log)
+
+Output grammars follow CppUTest v4.0's own source (TestOutput.cpp,
+TestRegistry.cpp): eclipse-style failure locations (the default),
+'-ln' space-separated 'Group.Name' enumeration.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .backend import TestId, TestOutcome
+
+# Suite argv for each backend operation. VERBOSE_ARGS produces the
+# run output parse()/parse_run() understand; LIST_ARGS produces the
+# enumeration parse_list() understands.
+VERBOSE_ARGS = "-v"
+LIST_ARGS = "-ln"
+
+# CppUTest verbose (-v) output. Test ids are 'Group.Name', matching
+# the executable's own -ln enumeration format.
+_RAN = re.compile(r"^(?:IGNORE_)?TEST\((\w+), (\w+)\)", re.M)
+# Eclipse-style failure header: '<file>:<line>: error: Failure in
+# TEST(<group>, <name>)'. When the failure is outside the test file,
+# a second '<file>:<line>: error:' location line follows with the
+# actual failure site; the tab-indented message lines come last.
+_FAILED = re.compile(
+    r"^(.*):(\d+): error: Failure in TEST\((\w+), (\w+)\)$", re.M)
+_LOCATION = re.compile(r"^(.*):(\d+): error:$")
+_SUMMARY = re.compile(r"^(?:OK|Errors) \(.*\)", re.M)
+_LIST_ID = re.compile(r"(\w+)\.(\w+)$")
+
+
+def list_argv():
+    """Suite argv that enumerates tests in parse_list()'s format."""
+    return LIST_ARGS
+
+
+def run_all_argv():
+    """Suite argv that runs every test in parse_run()'s format."""
+    return VERBOSE_ARGS
+
+
+def run_one_argv(group, name):
+    """Suite argv that runs exactly one test in parse_run()'s
+    format ('-sg'/'-sn' are CppUTest's strict-match filters)."""
+    return f"{VERBOSE_ARGS} -sg {group} -sn {name}"
+
+
+def parse_list(text):
+    """Parse '-ln' enumeration output: space-separated 'Group.Name'
+    tokens. Returns a list of TestId."""
+    text = text.replace("\r\n", "\n")
+    ids = []
+    for token in text.split():
+        match = _LIST_ID.fullmatch(token)
+        if not match:
+            raise ValueError(
+                f"not a CppUTest -ln test list:\n{text}")
+        ids.append(TestId(*match.groups()))
+    return ids
+
+
+def _failures(text):
+    """Map 'Group.Name' -> (file, line, message) for each failure
+    block in verbose output."""
+    lines = text.splitlines()
+    failures = {}
+    for match in _FAILED.finditer(text):
+        file, line, group, name = match.groups()
+        rest = lines[text.count("\n", 0, match.start()) + 1:]
+        if rest and _LOCATION.match(rest[0]):
+            # Failure outside the test file: the second location
+            # line carries the actual failure site.
+            file, line = _LOCATION.match(rest[0]).groups()
+            rest = rest[1:]
+        message = []
+        for text_line in rest:
+            if not text_line:
+                break
+            message.append(text_line.lstrip("\t"))
+        failures[f"{group}.{name}"] = (file, int(line), "\n".join(message))
+    return failures
+
+
+def parse_run(text):
+    """Parse verbose (-v) run output into a list of TestOutcome, in
+    output order. Raises ValueError when the output carries no
+    CppUTest summary line (the run died before finishing). DOS logs
+    arrive with CRLF line endings; both endings are accepted."""
+    text = text.replace("\r\n", "\n")
+    if not _SUMMARY.search(text):
+        raise ValueError(
+            f"no CppUTest summary line in suite output:\n{text}")
+    failures = _failures(text)
+    outcomes = []
+    for group, name in _RAN.findall(text):
+        file, line, message = failures.get(f"{group}.{name}", ("", 0, ""))
+        outcomes.append(TestOutcome(
+            group=group, name=name,
+            passed=f"{group}.{name}" not in failures,
+            file=file, line=line, message=message))
+    return outcomes
+
+
+def parse(text):
+    """Normalize CppUTest verbose output: return a dict with "ran" and
+    "failed" (sets of 'Group.Name' ids) and "summary" (the OK/Errors
+    line)."""
+    outcomes = parse_run(text)
+    return {
+        "ran": {f"{o.group}.{o.name}" for o in outcomes},
+        "failed": {f"{o.group}.{o.name}" for o in outcomes
+                   if not o.passed},
+        "summary": _SUMMARY.search(text).group(0),
+    }

testaferro/facade.py

@@ -0,0 +1,227 @@
+# SPDX-FileCopyrightText: 2026 Paul Galbraith
+# SPDX-License-Identifier: BSD-3-Clause
+"""The pytest facade: surface a guest suite's tests as pytest items.
+
+A consumer's test module hands over a reference to its suite
+executable (public entry point: testaferro.guest_suite):
+
+    test_guest_case = testaferro.guest_suite(HERE / "SUITE.EXE")
+
+The executable is interrogated to select the matching platform binding
+(currently: DOS programs, run under QEMU); a provably unsupported
+binary is rejected with a clear error. Alternatively a prebuilt
+Backend may be passed in place of the path — the custom execution
+escape hatch and test seam for this facade.
+
+Each of the suite's tests becomes one pytest item, so pytest's own
+selection (-k, node ids) drives what actually runs remotely. When the
+whole suite is selected, the facade batches everything into a single
+run_all(); a narrowed selection falls back to individual run_test()
+calls. Remote failures are replayed with the
+guest side's original file/line/message rather than a traceback into
+this facade.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+
+from . import binfmt
+from .backend import TestId
+
+# platform name -> binding module (a sibling of this one), imported only
+# when dispatch selects it
+_PLATFORM_BINDINGS = {"dos": "qemu"}
+
+
+class ResultBroker:
+    """Lazily fetch outcomes for one suite, deciding between one
+    batched run_all() and per-test run_test() calls from how much of
+    the suite pytest actually selected. pytest-free on purpose."""
+
+    def __init__(self, backend, ids):
+        self._backend = backend
+        self._ids = list(ids)
+        self._batch = None
+        self._single = {}
+
+    def outcome(self, test_id, selected_ids):
+        if set(selected_ids) == set(self._ids):
+            if self._batch is None:
+                self._batch = {TestId(o.group, o.name): o
+                               for o in self._backend.run_all()}
+            if test_id not in self._batch:
+                raise LookupError(
+                    f"target did not run test {test_id} "
+                    "(host and target test lists out of sync?)")
+            return self._batch[test_id]
+        if test_id not in self._single:
+            self._single[test_id] = self._backend.run_test(
+                test_id.group, test_id.name)
+        return self._single[test_id]
+
+
+def guest_suite(target, framework=None, enumerator=None,
+               platform=None, machine=None, **machine_options):
+    """Return a pytest test function with one parameterized item per
+    test in the referenced suite. Assign it to a test_-prefixed
+    module attribute:
+
+        test_guest_case = testaferro.guest_suite(HERE / "SUITE.EXE")
+
+    `target` is a path to the suite executable (interrogated to pick
+    the guest backend) or a prebuilt Backend. The keyword options
+    apply to the path form only: `framework` overrides the framework
+    adapter, `enumerator` supplies a faster host-side source of the
+    test list, `platform` chooses an OS platform when the executable's
+    own format should not decide, and `machine` chooses a named test
+    machine declared with testaferro.config() or testaferro.ini
+    (searched upward from this call site). Any further keyword is
+    machine-specific and validated by the selected binding: today,
+    `boot_image=` or `machine_config=` for DOS.
+
+    Enumeration (backend.list_tests()) happens in its own session at
+    import/collection time. A second session starts lazily when the
+    first selected item runs and is cleaned up when pytest finishes.
+
+    The returned function is re-homed to the caller's file and call
+    line, so IDE per-item actions that key on source location (run
+    one item, jump to source) resolve to the consumer's module
+    rather than this facade.
+    """
+    import inspect
+
+    import pytest
+
+    frame = inspect.currentframe()
+    caller = None if frame is None else frame.f_back
+    call_site = (None if caller is None else
+                 (caller.f_code.co_filename, caller.f_lineno))
+    del frame, caller
+
+    options = {name: value
+               for name, value in (("framework", framework),
+                                   ("enumerator", enumerator))
+               if value is not None}
+    options.update(machine_options)
+    if isinstance(target, (str, os.PathLike)):
+        search_from = (None if call_site is None
+                       else os.path.dirname(call_site[0]))
+        backend = _dispatched_backend(
+            target, platform, machine, options,
+            search_from=search_from)
+    else:
+        given = sorted(options)
+        if platform is not None:
+            given.append("platform")
+        if machine is not None:
+            given.append("machine")
+        if given:
+            raise TypeError(
+                "keyword options apply only when passing an "
+                "executable path; a prebuilt Backend carries its own "
+                "configuration: " + ", ".join(given))
+        backend = target
+
+    try:
+        backend.start_session()
+        ids = list(backend.list_tests())
+    finally:
+        backend.stop_session()
+    broker = ResultBroker(backend, ids)
+    execution_session_started = False
+
+    def start_execution_session(config):
+        nonlocal execution_session_started
+        if execution_session_started:
+            return
+        try:
+            backend.start_session()
+            config.add_cleanup(backend.stop_session)
+        except BaseException:
+            backend.stop_session()
+            raise
+        execution_session_started = True
+
+    @pytest.mark.parametrize("guest_test", ids, ids=_item_id)
+    def run_guest_test(guest_test, request):
+        start_execution_session(request.config)
+        try:
+            outcome = broker.outcome(guest_test,
+                                     _selected_ids(request, broker))
+        except LookupError as error:
+            pytest.fail(str(error), pytrace=False)
+        if not outcome.passed:
+            where = (f"{outcome.file}:{outcome.line}: "
+                     if outcome.file else "")
+            pytest.fail(f"guest test failed: {where}{outcome.message}",
+                        pytrace=False)
+
+    if call_site is not None:
+        run_guest_test.__code__ = run_guest_test.__code__.replace(
+            co_filename=call_site[0], co_firstlineno=call_site[1])
+    run_guest_test._testaferro_broker = broker
+    return run_guest_test
+
+
+def _dispatched_backend(target, platform, machine, options,
+                        search_from=None):
+    """Build the suite backend for an executable path: select the
+    platform binding — from the caller's machine/platform choice or
+    the executable's own format — import it, and hand it its options."""
+    from . import machines
+
+    machines.load_config(search_from=search_from)
+    if platform is not None:
+        platform = str(platform).lower()
+        if platform not in _PLATFORM_BINDINGS:
+            raise ValueError(f"unsupported platform {platform!r}; "
+                             "supported: "
+                             + ", ".join(sorted(_PLATFORM_BINDINGS)))
+    fmt = binfmt.classify(target)
+    if fmt.platform is None and platform is None and machine is None:
+        raise ValueError(
+            f"{os.path.basename(os.fspath(target))} is "
+            f"{fmt.kind} executable; no supported platform can run it")
+    selected = machines.select(machine, platform, fmt.platform)
+    if selected is not None:
+        _, machine_config = selected
+        if "machine_config" in options:
+            raise TypeError(
+                "machine_config cannot be combined with a named machine")
+        selected_platform = machine_config.platform
+        options["machine_config"] = machine_config
+    else:
+        selected_platform = platform if platform is not None else fmt.platform
+    if selected_platform not in _PLATFORM_BINDINGS:
+        raise ValueError(f"unsupported platform {selected_platform!r}; "
+                         "supported: "
+                         + ", ".join(sorted(_PLATFORM_BINDINGS)))
+    binding = importlib.import_module(
+        "." + _PLATFORM_BINDINGS[selected_platform], __package__)
+    try:
+        return binding.suite_backend(target, **options)
+    except TypeError as error:
+        if "unexpected keyword argument" not in str(error):
+            raise
+        raise TypeError(f"{error} — options are machine-specific; "
+                        f"the selected platform is "
+                        f"{selected_platform!r}") from None
+
+
+def _item_id(test_id):
+    """The bracketed id for one guest test item. A dash join, not
+    str(TestId): a dot inside a parametrize id breaks IDE
+    tree→target mapping (dots are hierarchy separators there),
+    turning run-this-item into run-the-whole-file."""
+    return f"{test_id.group}-{test_id.name}"
+
+
+def _selected_ids(request, broker):
+    """The suite's test ids that survived pytest's selection —
+    exactly the items pytest is going to run this session."""
+    return [item.callspec.params["guest_test"]
+            for item in request.session.items
+            if getattr(item.function, "_testaferro_broker", None)
+            is broker]