py-wlcli 0.8.0__py3-none-any.whl

wl_cli/commands/testcommands/test_runner.py

@@ -0,0 +1,1343 @@
+"""Test runner module for test commands."""
+
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from collections.abc import Callable, Iterator
+from pathlib import Path
+from xml.etree import ElementTree
+
+from ...utils.logging import log_info_pair
+from ...utils.output_context import is_concise_mode
+from ...utils.run_evidence import TestRunEvidence
+from .batch_processor import BatchProcessor
+from .cache_manager import CacheManager
+from .report_generator import ReportGenerator
+from .single_test_executor import SingleTestExecutor
+
+# ---------------------------------------------------------------------------
+# Per-file attribution: which user passthrough args travel along (#183)
+#
+# The batch run produced its verdict *under these args*, so a per-file re-run
+# that drops them can reach a different verdict for a file the user's selection
+# had effectively passed: a failing test the user excluded with `-k` would
+# still mark its file failed and drop it from the cache. Selection args
+# therefore have to travel. Only the subset below does -- an allowlist, not
+# "everything dash-shaped", because a half-forwarded option is worse than a
+# dropped one: `-k` without its expression swallows the test path, pytest then
+# collects nothing, exits 5, and `_process_test_result` reads that as success,
+# caching a file that never ran.
+#
+# Two families are deliberately *out* of the allowlist (rv183):
+#   * warning filters (`-W` / `--pythonwarnings`): a single-file run does not
+#     reproduce the batch run's warning stack, so `-W error` here can fail a
+#     file the batch run passed -- attribution would drop a good file.
+#   * stateful selection (`--lf` / `--ff` / `--nf` / `--sw` and long forms):
+#     they read *and rewrite* `.pytest_cache` in the project root, so file N+1
+#     sees state file N left behind -- not the batch selection at all. It also
+#     cannot be forwarded completely (`--lfnf` is the missing half).
+# The companion guard for the `rc == 5` case lives in
+# `SingleTestExecutor.execute`: "no tests ran under a forwarded selection" is
+# not evidence of a pass, so the file is left unattributed instead of cached.
+# ---------------------------------------------------------------------------
+
+#: Option spelling -> whether it consumes the following token as its value.
+_ATTRIBUTION_SAFE_OPTIONS: dict[str, bool] = {
+    # Selection: the filter the batch verdict was produced under.
+    "-k": True,
+    "--keyword": True,
+    "-m": True,
+    "--markexpr": True,
+    "--deselect": True,
+    "--ignore": True,
+    "--ignore-glob": True,
+    # Failure / output switches. None of them can change a single file's
+    # pass/fail verdict: `-x` only stops earlier *within* a file that has
+    # already failed.
+    "-x": False,
+    "--exitfirst": False,
+    "--maxfail": True,
+    "-q": False,
+    "--quiet": False,
+    "-v": False,
+    "--verbose": False,
+    "-r": True,
+    "--tb": True,
+    "-l": False,
+    "--showlocals": False,
+    "--strict-markers": False,
+    "--strict-config": False,
+    # Configuration the batch run honoured.
+    "-o": True,
+    "--override-ini": True,
+    "-p": True,
+    "--plugin": True,
+    "-c": True,
+    "--config-file": True,
+}
+
+
+def _attribution_args(passed_args: list[str]) -> list[str]:
+    """Return the user passthrough args a per-file attribution run may reuse.
+
+    Forward only :data:`_ATTRIBUTION_SAFE_OPTIONS` and the values of the ones
+    that take a value. Deliberately *not* forwarded:
+
+    * positional selectors (``tests/subdir``, ``tests/x.py::TestA``): the
+      subprocess is already pinned to exactly one file, and re-adding a path
+      would run other files too -- their failures would be attributed to this
+      one.
+    * coverage flags (``--cov*``, ``--no-cov``): this pass exists to attribute
+      cache, not to collect coverage; the single-file executor already decides
+      ``--no-cov`` for the interpreter that runs it.
+    * report artifacts (``--junitxml``, ``--html``, ``--cov-report``,
+      ``--pastebin``): every file would overwrite the same artifact.
+    * warning filters (``-W``, ``--pythonwarnings``): the single-file command
+      does not reproduce the batch warning stack, so forwarding can fail a file
+      the batch run passed (rv183 M2).
+    * stateful selection (``--lf``, ``--ff``, ``--nf``, ``--sw``): their state
+      lives in ``.pytest_cache``, which each per-file run rewrites; the series
+      is also incomplete without ``--lfnf`` (rv183 M3).
+    * anything unrecognised: dropped whole, rather than half-forwarded.
+    """
+    forwarded: list[str] = []
+    index = 0
+    while index < len(passed_args):
+        arg = passed_args[index]
+        index += 1
+
+        if not arg.startswith("-"):
+            continue  # positional selector, see above
+
+        name, separator, _value = arg.partition("=")
+        takes_value = _ATTRIBUTION_SAFE_OPTIONS.get(name)
+        if takes_value is None:
+            # `-kexpr` / `-Werror`: a short option carrying its value inline.
+            if len(arg) > 2 and not arg.startswith("--"):
+                takes_value = _ATTRIBUTION_SAFE_OPTIONS.get(arg[:2])
+                if takes_value:
+                    forwarded.append(arg)
+            continue
+
+        if not takes_value or separator:
+            forwarded.append(arg)
+            continue
+
+        # Never emit a value-taking option without its value: a dangling `-k`
+        # would consume the test path and leave nothing to run.
+        if index < len(passed_args) and not passed_args[index].startswith("-"):
+            forwarded.append(arg)
+            forwarded.append(passed_args[index])
+            index += 1
+    return forwarded
+
+
+# ---------------------------------------------------------------------------
+# Executed-file derivation (#191)
+#
+# A pytest process that exits 0 proves something about *the tests it ran*, and
+# nothing about the rest of the tree. The cache used to record every file under
+# tests/ after such a run, so `wl test --use-cache tests/test_a.py` marked a red
+# tests/test_b.py as passing and the next `wl test -s` reported the suite green
+# from the cache. The write-back set is now derived here -- and when it cannot
+# be derived, nothing is written (fail-closed).
+# ---------------------------------------------------------------------------
+
+#: Directories a selector walk never descends into. A directory selector like
+#: `.` must not turn a virtualenv's vendored `test_*.py` files into "tests this
+#: run executed" (rv191 F3), and walking them is pure cost.
+_SKIP_DIRECTORY_NAMES: frozenset[str] = frozenset(
+    {"__pycache__", "site-packages", "node_modules", "venv", "build", "dist"}
+)
+
+#: Passthrough options a run may carry *and still write cache* -- an allowlist,
+#: not a blocklist. Two rounds of blocklists taught the lesson (rv191: first
+#: `--collect-only`, then `--setup-only`): every enumeration missed the next
+#: pytest flag that changes what a green exit code means. Here the default is
+#: "denied", so a new or unknown switch can only cost a cache write, never
+#: produce a false green.
+#:
+#: Everything listed leaves a green exit code meaning what it says: the tests
+#: the run selected did pass. Output/verbosity, fail-fast switches (a green `-x`
+#: run still ran every selected test, and any failure would be non-green),
+#: collection strictness (which can only turn a pass into an error), coverage
+#: instrumentation, and parallelism (`-n`/`--dist` move where a test runs, not
+#: whether it runs).
+#: Deliberately absent: warning filters (`-W`/`--pythonwarnings`, see below),
+#: selection filters (`-k`, `-m`, `--deselect`, `--ignore*`,
+#: `--lf`/`--ff`/`--nf`/`--sw`), loading-chain switches (`--noconftest`, `-p`,
+#: `-c`, `-o`, `--rootdir`, `--confcutdir`, `--import-mode`), non-executing
+#: modes (`--collect-only`, `--setup-only`, `--setup-plan`, `--version`,
+#: `--fixtures`, `--markers`, `--help`, `--trace-config`), report destinations
+#: (`--junitxml`/`--junit-xml`, `--html`, `--pastebin`) and seed/order flags
+#: (`--randomly-seed`).
+#:
+#: Warning filters are denied in both directions (#199 rv199). A filter can
+#: *weaken* the verdict -- with `filterwarnings = error` in the project config,
+#: `-W ignore` turns a red file green -- and the cache key cannot see a
+#: command-line switch, so storing that green under the plain key let the next
+#: unfiltered run answer "using cache" over a suite that is red. Storing only
+#: under "stricter" filters was the earlier idea and it does not hold: only a
+#: bare `error` is stricter than the project's own setting, and a filtered run
+#: can never read cache anyway, so the write bought one hit for a plainer run --
+#: exactly the unsound direction. `-W` stays in
+#: :data:`_VALUE_TAKING_OPTIONS` so its value is not mistaken for a selector.
+#:
+#: Short-option spellings matter here too (#199): `-rA`, `-vv` and `-n4` are
+#: the same switches as `-r A`, `-v -v` and `-n 4`, so
+#: :func:`_short_option_name` reduces them to their two-character name before
+#: this set is consulted. Keeping the inline spellings denied only meant that
+#: `wl test -- -rA` and `wl test -- -vv` (ordinary habits) silently stopped
+#: caching.
+_CACHE_SAFE_OPTIONS: frozenset[str] = frozenset(
+    {
+        # Output and verbosity.
+        "-q",
+        "--quiet",
+        "-v",
+        "--verbose",
+        "--no-header",
+        "--no-summary",
+        "--disable-warnings",
+        "-r",
+        "--tb",
+        "-l",
+        "--showlocals",
+        "--color",
+        "--code-highlight",
+        "--durations",
+        "--durations-min",
+        "--log-level",
+        "--log-file",
+        "--log-format",
+        "--log-date-format",
+        "-s",
+        "--capture",
+        # Fail-fast: green still means every selected test ran and passed.
+        "-x",
+        "--exitfirst",
+        "--maxfail",
+        # Collection strictness: can only turn a pass into an error.
+        "--strict-markers",
+        "--strict-config",
+        # Parallelism: changes where a test runs, not whether it runs.
+        "-n",
+        "--numprocesses",
+        "--dist",
+    }
+)
+
+#: Prefix allowlist for the coverage family (`--cov`, `--cov-report=...`,
+#: `--cov-fail-under=...`, `--no-cov`, `--no-cov-on-fail`). Coverage changes
+#: instrumentation and exit status, not a test's verdict.
+_CACHE_SAFE_OPTION_PREFIXES: tuple[str, ...] = ("--cov", "--no-cov")
+
+#: Spellings of pytest's junit report destination. Both must be recognised: the
+#: runner adds a report of its own to observe the run, and writing that over a
+#: user-supplied path would silently discard their file (rv191 F2).
+_JUNIT_REPORT_OPTIONS: frozenset[str] = frozenset({"--junitxml", "--junit-xml"})
+
+#: Options that consume the following token as their value. Used only to keep
+#: an option's value from being mistaken for a path selector; deliberately a
+#: superset of the attribution allowlist, because an unrecognised value option
+#: would otherwise turn its value into a selector that does not exist.
+_VALUE_TAKING_OPTIONS: frozenset[str] = frozenset(
+    {
+        *(
+            name
+            for name, takes_value in _ATTRIBUTION_SAFE_OPTIONS.items()
+            if takes_value
+        ),
+        "-W",
+        "--pythonwarnings",
+        "--durations",
+        "--durations-min",
+        "--rootdir",
+        "--confcutdir",
+        "--junitxml",
+        "--junit-xml",
+        "--junit-prefix",
+        "--color",
+        "--capture",
+        "-n",
+        "--numprocesses",
+        "--dist",
+        "--timeout",
+        "--basetemp",
+        "--import-mode",
+        "--log-level",
+        "--log-file",
+        "--log-format",
+        "--log-date-format",
+        "-a",
+        "--alluredir",
+        "--randomly-seed",
+    }
+)
+
+
+def _short_option_name(arg: str) -> str | None:
+    """The two-character name of an inline short option, or ``None``.
+
+    ``-rA``, ``-Werror`` and ``-n4`` carry their value in the same token;
+    ``-vv``/``-qq`` repeat a flag. In every case pytest reads the option as
+    ``arg[:2]`` plus a remainder that is a value or a repetition, so the
+    allowlist must be asked about the two-character name -- asking about the
+    raw token denied ordinary habits like ``-rA`` and ``-vv`` (#199). The
+    remainder never changes whether the run executes tests: for a value-taking
+    option it *is* the value, and for a flag it is the flag repeated.
+    """
+    if len(arg) <= 2 or arg.startswith("--"):
+        return None
+    return arg[:2]
+
+
+def _split_passthrough_args(passed_args: list[str]) -> tuple[list[str], list[str]]:
+    """Split the user's passthrough args into (option names, path selectors).
+
+    Option values are skipped using :data:`_VALUE_TAKING_OPTIONS`; an unknown
+    option's would-be value stays a selector, which then fails to resolve and
+    makes the whole selection "unknown" -- the safe direction.
+
+    Inline short options are reported by their canonical two-character name
+    (:func:`_short_option_name`), so ``-rA`` is vetted as ``-r``.
+    """
+    option_names: list[str] = []
+    selectors: list[str] = []
+    index = 0
+    while index < len(passed_args):
+        arg = passed_args[index]
+        index += 1
+        if not arg.startswith("-") or arg == "-":
+            selectors.append(arg)
+            continue
+
+        name, separator, _value = arg.partition("=")
+        if separator:
+            option_names.append(name)
+            continue
+        if name in _VALUE_TAKING_OPTIONS:
+            option_names.append(name)
+            if index < len(passed_args):
+                index += 1  # this token is the option's value, not a selector
+            continue
+        short = _short_option_name(arg)
+        option_names.append(short if short is not None else name)
+    return option_names, selectors
+
+
+def _selector_spelling(path: Path, project_root: str) -> str:
+    """A path as the user would spell it on the command line."""
+    try:
+        relative = path.relative_to(project_root)
+    except ValueError:
+        relative = path
+    return str(relative).replace("\\", "/")
+
+
+def _selector_path(selector: str, project_root: str) -> Path:
+    """Resolve a selector the way pytest does: relative to the project root."""
+    path = Path(selector)
+    if not path.is_absolute():
+        path = Path(project_root) / path
+    return path
+
+
+def _files_for_selector(
+    token: str, known: dict[str, Path], project_root: str
+) -> list[Path] | None:
+    """Resolve one already-normalised path selector to the files it names.
+
+    Returns ``None`` when the selector names nothing this run can vouch for;
+    the caller turns that into "the file set is unknown".
+    """
+    match = known.get(token)
+    if match is not None:
+        return [match]
+
+    if any(spelling == token or spelling.startswith(f"{token}/") for spelling in known):
+        # A directory selector: every known file underneath it.
+        return [
+            test_file
+            for spelling, test_file in known.items()
+            if spelling.startswith(f"{token}/")
+        ]
+
+    resolved = _selector_path(token, project_root)
+    if resolved.is_dir():
+        # Only files the project's own test tree knows about: a directory
+        # selector covers `.`, and `.` is where a virtualenv keeps thousands of
+        # vendored `test_*.py` files that were never this project's tests
+        # (rv191 F3).
+        chosen = [
+            known[spelling]
+            for discovered in _iter_python_files(resolved)
+            for spelling in (_selector_spelling(discovered, project_root),)
+            if spelling in known
+        ]
+        return chosen or None
+    if resolved.is_file():
+        return [resolved]
+    return None
+
+
+def _iter_python_files(directory: Path) -> Iterator[Path]:
+    """Yield ``*.py`` files under *directory*, skipping vendored trees.
+
+    Hidden directories (``.venv``, ``.git``, ...) and the usual dependency
+    directories are pruned rather than filtered afterwards, so a selector like
+    `.` costs one walk of the project's own sources.
+    """
+    for current, dirnames, filenames in os.walk(directory):
+        dirnames[:] = sorted(
+            name
+            for name in dirnames
+            if not name.startswith(".") and name not in _SKIP_DIRECTORY_NAMES
+        )
+        for name in sorted(filenames):
+            if name.endswith(".py") and name != "__init__.py":
+                yield Path(current) / name
+
+
+def _executed_test_files(
+    passed_args: list[str], project_root: str, all_test_files: list[Path]
+) -> list[Path] | None:
+    """The test files this run executed, or ``None`` when that is unknown.
+
+    ``None`` means "cannot be determined" and the caller must not write cache:
+    a green exit code from a run whose file set is unknown is not evidence
+    about any particular file. Concretely this covers a filtered run
+    (``-k``/``-m``/``--deselect``/... -- the verdict is about a subset of each
+    file's tests), a run whose loading chain or execution mode was switched
+    (``--noconftest``, ``-p``, ``--collect-only``, ...), a nodeid selector
+    (``tests/test_a.py::TestA`` -- a slice of a file), and a selector that does
+    not resolve to anything.
+
+    With no positional selector the default selection is the whole ``tests/``
+    tree, which is exactly ``all_test_files``.
+
+    ``PYTEST_ADDOPTS`` is not consulted for the selection: an addopts that
+    could narrow it (a filter or a positional selector) makes the whole run
+    uncacheable in :func:`_addopts_cache_blocker` (#199), and an addopts that
+    survives that gate contains only switches that cannot select.
+    """
+    option_names, selectors = _split_passthrough_args(passed_args)
+    if _unsafe_option(option_names) is not None:
+        return None
+    if not selectors:
+        return list(all_test_files)
+
+    known = {
+        _selector_spelling(test_file, project_root): test_file
+        for test_file in all_test_files
+    }
+
+    executed: list[Path] = []
+    seen: set[Path] = set()
+    for selector in selectors:
+        if "::" in selector:
+            # A nodeid ran part of the file; a whole-file pass was never proven.
+            return None
+        token = selector.replace("\\", "/")
+        while token.startswith("./"):
+            token = token[2:]
+        token = token.rstrip("/")
+
+        chosen = _files_for_selector(token, known, project_root)
+        if chosen is None:
+            return None
+        for path in chosen:
+            if path not in seen:
+                seen.add(path)
+                executed.append(path)
+    return executed
+
+
+def _unsafe_option(option_names: list[str]) -> str | None:
+    """The first passthrough option that is not on the cache-safe allowlist.
+
+    ``None`` means every option in the run is vetted. Anything else -- a
+    selection filter, a loading-chain switch, a collection-only mode, or a flag
+    nobody has thought about yet -- makes the run uncacheable.
+    """
+    for name in option_names:
+        if name in _CACHE_SAFE_OPTIONS:
+            continue
+        if name.startswith(_CACHE_SAFE_OPTION_PREFIXES):
+            continue
+        return name
+    return None
+
+
+def _cache_write_blocker(option_names: list[str]) -> str | None:
+    """Why this run may not write cache at all, or ``None`` when it may."""
+    unsafe = _unsafe_option(option_names)
+    if unsafe is None:
+        return None
+    return (
+        f"the run used {unsafe}, which is not on the cache-safe allowlist of "
+        "pytest switches"
+    )
+
+
+def _addopts_cache_blocker(env: dict[str, str] | None) -> str | None:
+    """Why ``PYTEST_ADDOPTS`` makes this run uncacheable, or ``None``.
+
+    pytest prepends ``PYTEST_ADDOPTS`` to *every* command line, so it is a
+    second door through which a switch reaches the run -- and until #199 it was
+    the unwatched one: the cache key included the variable's value, but nothing
+    checked what the value *meant*. ``PYTEST_ADDOPTS=--collect-only`` therefore
+    exited 0 without executing a test while the allowlist, which only ever saw
+    the command line, raised no objection.
+
+    The value is split exactly as pytest splits it (``shlex.split``) and vetted
+    against the same allowlist. A positional token counts as unsafe here even
+    though the command line treats it as a selector: addopts selectors would
+    narrow the run behind the runner's back, so its file set is unknowable.
+    An unparsable value is denied outright.
+    """
+    raw = (env or {}).get("PYTEST_ADDOPTS", "")
+    if not raw.strip():
+        return None
+    try:
+        tokens = shlex.split(raw)
+    except ValueError:
+        return (
+            "PYTEST_ADDOPTS cannot be parsed, so the run it produces cannot be vetted"
+        )
+    option_names, selectors = _split_passthrough_args(tokens)
+    unsafe = _unsafe_option(option_names)
+    if unsafe is not None:
+        return (
+            f"PYTEST_ADDOPTS injects {unsafe}, which is not on the cache-safe "
+            "allowlist of pytest switches"
+        )
+    if selectors:
+        return (
+            "PYTEST_ADDOPTS injects a positional selector, so this run's test "
+            "files cannot be determined"
+        )
+    return None
+
+
+def _execution_report(
+    passed_args: list[str], report_mode: bool
+) -> tuple[Path | None, bool]:
+    """Where to read this run's junit report, and whether we own the file.
+
+    ``(None, False)`` means the run cannot be observed: the user asked for
+    their own junit report (``--junitxml`` or its ``--junit-xml`` alias), and
+    writing ours over their path would silently discard their file -- losing
+    the cache for one run is the cheaper mistake. In report mode the command
+    builder already writes ``todos/junit.xml``; otherwise the runner points
+    pytest at a fresh temporary file, so a leftover report can never be read as
+    this run's.
+
+    The report is the *positive signal* the cache write needs (rv191 F1): pytest
+    writes one test case per executed test, and writes none for
+    ``--collect-only``, ``--version`` or ``--fixtures``.
+    """
+    option_names, _selectors = _split_passthrough_args(passed_args)
+    if _JUNIT_REPORT_OPTIONS.intersection(option_names):
+        return None, False
+    if report_mode:
+        # TestCommandBuilder's report-mode target, relative to the process CWD.
+        return Path("todos") / "junit.xml", False
+    handle, name = tempfile.mkstemp(prefix="wl-test-report-", suffix=".xml")
+    os.close(handle)
+    return Path(name), True
+
+
+def _case_spelling(case: ElementTree.Element, project_root: str) -> str | None:
+    """The ``file`` attribute of a junit ``<testcase>``, normalised."""
+    file_attr = case.get("file")
+    if not file_attr:
+        return None
+    token = file_attr.replace("\\", "/")
+    while token.startswith("./"):
+        token = token[2:]
+    if Path(token).is_absolute():
+        return _selector_spelling(Path(token), project_root)
+    return token
+
+
+def _match_classname(classname: str, by_module: dict[str, Path]) -> Path | None:
+    """Match a junit ``classname`` to a known test file by module prefix.
+
+    pytest's junit report often omits ``file``; the ``classname`` is the module
+    path (``tests.test_a``), optionally suffixed by the test class. Longest
+    prefix wins, so ``pkg.tests.test_a`` is not shadowed by ``pkg.tests``.
+    """
+    for prefix, test_file in by_module.items():
+        if classname == prefix or classname.startswith(f"{prefix}."):
+            return test_file
+    return None
+
+
+def _report_outcome_counts_agree(report_root: ElementTree.Element) -> bool:
+    """Whether every ``<testcase>`` in the report carries a counted outcome.
+
+    pytest stamps each ``<testsuite>`` with ``tests``: the number of outcomes
+    the session recorded (passed + failed + skipped + error). It emits one
+    ``<testcase>`` element per item it *touched*. For a run that actually
+    executes its tests the two numbers agree -- verified against ``-x``, xdist,
+    teardown errors, xfail and parametrisation -- but ``--setup-only`` touches
+    every item while recording an outcome only for the items it skips, so its
+    ``tests`` count falls short of the testcase count.
+
+    That shortfall is the report's *own* proof that the run did not execute
+    what it lists. Checking it here makes the evidence defence independent of
+    the switch allowlist (#199): a ``--setup-only`` run is already denied
+    before it starts, but if a future whitelist change or a `PYTEST_ADDOPTS`
+    path ever let it through, this gate still refuses to read its report as
+    proof of execution. A report with no ``<testsuite>`` at all proves nothing.
+    """
+    suites = list(report_root.iter("testsuite"))
+    if not suites:
+        return False
+    for suite in suites:
+        declared = suite.get("tests")
+        if declared is None:
+            # pytest always writes it; a report without it cannot be verified.
+            return False
+        try:
+            declared_count = int(declared)
+        except ValueError:
+            return False
+        if declared_count != len(suite.findall("testcase")):
+            return False
+    return True
+
+
+def _executed_cases(report_root: ElementTree.Element) -> Iterator[ElementTree.Element]:
+    """The report's testcases that carry a verdict.
+
+    Skipped cases are dropped here: pytest records a skip when it evaluated a
+    skip marker or a missing dependency, so the body never ran and the file was
+    never proven to pass (#199). The junit XML cannot distinguish a skip raised
+    inside the body from one raised at setup, so the conservative reading is
+    the only safe one -- a skip is a decision, not a verdict.
+    """
+    return (
+        case for case in report_root.iter("testcase") if case.find("skipped") is None
+    )
+
+
+def _executed_files_from_report(
+    report_path: Path | None,
+    project_root: str,
+    known: dict[str, Path],
+    started_at: float,
+) -> list[Path]:
+    """The known test files that pytest's own report proves it executed.
+
+    Returns ``[]`` -- "no proof" -- for a missing, stale, empty or unparsable
+    report, for a report whose ``tests`` counts do not match its testcase
+    elements (:func:`_report_outcome_counts_agree`), and for a report with zero
+    test cases. A file named in the report but outside the project's test tree
+    is ignored: only files this project knows as tests may ever be cached
+    (rv191 F3).
+
+    A skipped testcase is not evidence either (#199): pytest records a skip
+    when it evaluated a skip marker or a missing dependency -- the test body
+    never ran -- so an all-skipped file would otherwise be cached as passing,
+    and would keep reading "passed" after the skip condition changed. The
+    junit XML does not distinguish a skip raised inside the body from one
+    raised at setup, so the conservative reading is the only safe one: a skip
+    is a decision, not a verdict.
+    """
+    if report_path is None:
+        return []
+    try:
+        stat = report_path.stat()
+    except OSError:
+        return []
+    if stat.st_size == 0 or stat.st_mtime < started_at - 1.0:
+        return []
+    try:
+        report_root = ElementTree.parse(report_path).getroot()
+    except (ElementTree.ParseError, OSError):
+        return []
+    if not _report_outcome_counts_agree(report_root):
+        return []
+
+    by_resolved: dict[Path, Path] = {}
+    for test_file in known.values():
+        try:
+            by_resolved[test_file.resolve()] = test_file
+        except OSError:  # pragma: no cover - unresolvable path, never matched
+            continue
+
+    by_module = {
+        spelling[: -len(".py")].replace("/", "."): test_file
+        for spelling, test_file in sorted(
+            known.items(), key=lambda item: len(item[0]), reverse=True
+        )
+        if spelling.endswith(".py")
+    }
+
+    executed: list[Path] = []
+    seen: set[Path] = set()
+    for case in _executed_cases(report_root):
+        spelling = _case_spelling(case, project_root)
+        matched: Path | None = None
+        if spelling is not None:
+            matched = known.get(spelling)
+            if matched is None:
+                try:
+                    matched = by_resolved.get(
+                        _selector_path(spelling, project_root).resolve()
+                    )
+                except OSError:  # pragma: no cover - unresolvable path
+                    matched = None
+        if matched is None:
+            matched = _match_classname(case.get("classname") or "", by_module)
+        if matched is None or matched in seen:
+            continue
+        seen.add(matched)
+        executed.append(matched)
+    return executed
+
+
+def _write_run_evidence(
+    path: Path | None,
+    *,
+    project_root: str,
+    started_at: float,
+    executed_files: list[Path],
+    cache_hits: int,
+    cache_total: int,
+    no_cache_requested: bool,
+    cacheable: bool,
+    cache_blocked_reason: str | None,
+    exit_code: int,
+) -> None:
+    """Write this run's structured evidence without disturbing the run.
+
+    ``--evidence-json`` is an observation channel, not part of ``wl test``'s
+    exit-code contract: a write failure is reported on stderr and otherwise
+    ignored, exactly as a human-readable line would be. The gate reads the
+    absence as red -- the fail-closed direction, since a run nobody could
+    observe is not a run the gate can vouch for (ADR-0010).
+
+    ``executed_files`` are rendered relative to *project_root* (the same
+    spelling the user sees), and ``cache_total`` is the denominator of the
+    "cache usage" fact: the files this run considered for cache reuse.
+    """
+    if path is None:
+        return
+    evidence = TestRunEvidence(
+        executed_files=tuple(
+            _selector_spelling(test_file, project_root) for test_file in executed_files
+        ),
+        cache_hits=cache_hits,
+        cache_total=cache_total,
+        no_cache_requested=no_cache_requested,
+        cacheable=cacheable,
+        cache_blocked_reason=cache_blocked_reason,
+        exit_code=exit_code,
+        started_at=started_at,
+        finished_at=time.time(),
+    )
+    try:
+        evidence.write(path)
+    except OSError as error:
+        print(
+            f"[Evidence] Failed to write run evidence to {path}: {error}",
+            file=sys.stderr,
+        )
+
+
+class TestRunner:
+    """Test runner for executing tests."""
+
+    def __init__(self, cache_manager: CacheManager | None = None):
+        """Initialize TestRunner with dependencies.
+
+        Args:
+            cache_manager: Optional shared CacheManager. If omitted, creates a new one.
+        """
+        self.cache_manager = cache_manager or CacheManager()
+        self.batch_processor = BatchProcessor(self.cache_manager)
+        self.single_test_executor = SingleTestExecutor(self.cache_manager)
+        self.report_generator = ReportGenerator()
+
+    def _run_all_tests(
+        self,
+        python_executable: str,
+        project_root: str,
+        env: dict[str, str],
+        report_mode: bool,
+        passed_args: list[str],
+        debug_mode: bool,
+        cache_report: Path | None = None,
+    ) -> subprocess.CompletedProcess:
+        """Run all tests and return the result.
+
+        ``cache_report`` is the path pytest should write its junit XML to so the
+        runner can tell *which tests executed* -- the positive signal a cache
+        write requires (rv191). It is ``None`` in report mode (the command
+        builder already writes ``todos/junit.xml``) and when the user owns the
+        ``--junitxml`` destination.
+        """
+        from .python_env_detector import PythonEnvDetector
+        from .test_command_builder import TestCommandBuilder
+
+        python_detector = PythonEnvDetector()
+        command_builder = TestCommandBuilder(python_detector)
+
+        cmd_args = command_builder.build_command_args(
+            report_mode,
+            passed_args,
+            debug_mode,
+            python_executable=python_executable,
+            env=env,
+            # 复用调用方已解析的 root，而不是让 builder 再解析一次（rv192b NIT）：
+            # 同一个值既是 pytest 的 cwd，也是 `--cov` 的目标，两者不可能指向
+            # 不同的项目。
+            project_root=Path(project_root),
+        )
+
+        # 解释器与执行用 env 在 build_command_args 内部就已确定：pytest-cov 探针
+        # 问的是同一个 `python_executable`、拿的是同一份 `env`，此处不再事后改写
+        # `cmd_args[0]`——那正是「判定者与执行者不是同一个解释器」的来路。
+
+        if cache_report is not None:
+            # 核心 pytest 自带的 junit XML：每个真正执行的用例写一条
+            # <testcase>，`--collect-only` / `--version` / `--fixtures` 则一条
+            # 也不写。缓存写回只认这份「确有执行」的正向信号（rv191 F1）。
+            cmd_args.append(f"--junitxml={cache_report}")
+
+        # In all modes, pass through stdout/stderr to preserve colors and formatting
+        result = subprocess.run(
+            cmd_args,
+            cwd=project_root,
+            check=False,
+            env=env,
+        )
+
+        return result
+
+    def _cache_write_scope(
+        self,
+        blocker: str | None,
+        selected_files: list[Path] | None,
+        executed: list[Path],
+    ) -> list[Path] | None:
+        """The files this run may record as passed, or ``None`` for "record none".
+
+        Three independent gates, any of which closes the write:
+
+        1. the run must be cacheable at all -- no filter, no loading-chain
+           switch, no collection-only mode, nothing injected via
+           ``PYTEST_ADDOPTS``;
+        2. its selected file set must be derivable from the passthrough args;
+        3. pytest's own report must be internally consistent (outcome count vs
+           testcase count) and name at least one executed, non-skipped test --
+           *executed* is the caller's already-derived reading of that report.
+
+        The result is the *intersection* of the selection and the evidence, so
+        neither a stale report nor an over-broad selector (`.` reaching into
+        `.venv`) can smuggle a file into the cache (rv191 F1/F3).
+        """
+        if blocker is not None:
+            print(f"[Cache] Not saving results: {blocker}.")
+            return None
+        if selected_files is None:
+            print(
+                "[Cache] Not saving results: this run's test files cannot be "
+                "determined."
+            )
+            return None
+        if not executed:
+            print(
+                "[Cache] Not saving results: the pytest report shows no executed test."
+            )
+            return None
+        executed_set = set(executed)
+        scope = [test_file for test_file in selected_files if test_file in executed_set]
+        if not scope:
+            print(
+                "[Cache] Not saving results: no selected file appears in the "
+                "executed set."
+            )
+            return None
+        return scope
+
+    def _get_all_test_files(self, project_root: str) -> list[Path]:
+        """Get all test files in the project."""
+        test_files: list[Path] = []
+        tests_dir = Path(project_root) / "tests"
+
+        if tests_dir.exists() and tests_dir.is_dir():
+            # Recursively find all Python files in tests directory
+            for file_path in tests_dir.rglob("*.py"):
+                # Skip __init__.py files
+                if file_path.name != "__init__.py":
+                    test_files.append(file_path)
+
+        return test_files
+
+    @staticmethod
+    def _display_test_path(test_file: Path, project_root: str) -> str:
+        """Path as shown on an attribution progress line.
+
+        Relative to the project root when the file lives under it (the usual
+        case), so ``[3/87] tests/test_foo.py`` names the file the way the user
+        does rather than echoing an absolute temporary path.
+        """
+        try:
+            return str(test_file.relative_to(project_root))
+        except ValueError:
+            return str(test_file)
+
+    @staticmethod
+    def _step_cache_disabled(no_cache: bool, env: dict[str, str] | None) -> bool:
+        """Whether this step-by-step run must bypass the cache entirely.
+
+        Step mode walks the files itself and runs each one through
+        :class:`SingleTestExecutor`, so it inherited none of the normal path's
+        vetting: a ``PYTEST_ADDOPTS=--collect-only`` run executed nothing,
+        exited 0, and still recorded every file as passed (#199 rv199). The
+        same :func:`_addopts_cache_blocker` gate now covers this mode, and a
+        blocked value closes *both* directions of the cache -- no read, no
+        write -- exactly as it does in :meth:`_run_tests_normal`.
+        """
+        return no_cache or _addopts_cache_blocker(env) is not None
+
+    def _run_specified_tests_step_by_step(
+        self,
+        python_executable: str,
+        project_root: str,
+        env: dict[str, str],
+        debug_mode: bool,
+        no_cache: bool,
+        quiet: bool,
+        verbose: bool,
+        test_files: list[Path],
+        no_skip: bool = False,
+        evidence_path: Path | None = None,
+    ):
+        """Run specified test files in step-by-step mode."""
+        cache_disabled = self._step_cache_disabled(no_cache, env)
+        concise = is_concise_mode(override_quiet=quiet, override_verbose=verbose)
+
+        def run_one(test_file: Path) -> tuple[bool, str]:
+            return self.single_test_executor.execute(
+                python_executable,
+                test_file,
+                project_root,
+                env,
+                debug_mode,
+                cache_disabled,
+                step_by_step=True,
+                concise=concise,
+                no_skip=no_skip,
+            )
+
+        def run_batch(execute_single: Callable[[Path], tuple[bool, str]]) -> None:
+            self.batch_processor.process_specified_files(
+                quiet,
+                verbose,
+                test_files,
+                python_executable,
+                project_root,
+                env,
+                debug_mode,
+                cache_disabled,
+                execute_single,
+            )
+
+        self._run_step_batch_with_evidence(
+            run_one,
+            run_batch,
+            test_files=test_files,
+            project_root=project_root,
+            env=env,
+            no_cache=no_cache,
+            evidence_path=evidence_path,
+        )
+
+    def _run_tests_step_by_step(
+        self,
+        python_executable: str,
+        project_root: str,
+        env: dict[str, str],
+        debug_mode: bool,
+        no_cache: bool,
+        quiet: bool,
+        verbose: bool,
+        no_skip: bool = False,
+        evidence_path: Path | None = None,
+    ):
+        """Run tests in step-by-step mode, stopping at first error."""
+        test_files = self._get_all_test_files(project_root)
+        if not test_files:
+            # Fail closed (#191): "nothing to run" is not "everything passed".
+            # The quality gate drives this path; returning normally here made a
+            # project whose tests were never discovered exit 0 and look green.
+            print("No test files found.")
+            # The evidence still records what happened: no file ran, and the
+            # run terminates non-zero. A gate that requested evidence must not
+            # have to guess why the file is absent (#212).
+            blocker = _addopts_cache_blocker(env)
+            _write_run_evidence(
+                evidence_path,
+                project_root=project_root,
+                started_at=time.time(),
+                executed_files=[],
+                cache_hits=0,
+                cache_total=0,
+                no_cache_requested=no_cache,
+                cacheable=blocker is None,
+                cache_blocked_reason=blocker,
+                exit_code=1,
+            )
+            sys.exit(1)
+
+        cache_disabled = self._step_cache_disabled(no_cache, env)
+        concise = is_concise_mode(override_quiet=quiet, override_verbose=verbose)
+
+        def run_one(test_file: Path) -> tuple[bool, str]:
+            return self.single_test_executor.execute(
+                python_executable,
+                test_file,
+                project_root,
+                env,
+                debug_mode,
+                cache_disabled,
+                step_by_step=True,
+                concise=concise,
+                no_skip=no_skip,
+            )
+
+        def run_batch(execute_single: Callable[[Path], tuple[bool, str]]) -> None:
+            self.batch_processor.process_all_files(
+                quiet,
+                verbose,
+                test_files,
+                python_executable,
+                project_root,
+                env,
+                debug_mode,
+                cache_disabled,
+                execute_single,
+            )
+
+        self._run_step_batch_with_evidence(
+            run_one,
+            run_batch,
+            test_files=test_files,
+            project_root=project_root,
+            env=env,
+            no_cache=no_cache,
+            evidence_path=evidence_path,
+        )
+
+    def _run_step_batch_with_evidence(
+        self,
+        run_one: Callable[[Path], tuple[bool, str]],
+        run_batch: Callable[[Callable[[Path], tuple[bool, str]]], None],
+        *,
+        test_files: list[Path],
+        project_root: str,
+        env: dict[str, str],
+        no_cache: bool,
+        evidence_path: Path | None,
+    ) -> None:
+        """Run one step-mode batch and write its single aggregated evidence.
+
+        Step mode calls ``sys.exit(1)`` on the first failing file, so a
+        terminating path arrives here as ``SystemExit``. The exit code is
+        captured and the record written in ``finally``: one file per
+        invocation, aggregating every file the batch processed. Files served
+        from the cache are *not* "executed" -- the record lists exactly the
+        files the executor actually ran, derived from the run's own hit set.
+        """
+        started_at = time.time()
+        attempted: list[Path] = []
+        attempted_lock = threading.Lock()
+
+        def execute(test_file: Path) -> tuple[bool, str]:
+            # Recording before the call: the file was handed to the executor,
+            # so it is at worst a near-miss, and a cache hit is filtered out
+            # below from the run's own hit set.
+            with attempted_lock:
+                attempted.append(test_file)
+            return run_one(test_file)
+
+        blocker = _addopts_cache_blocker(env)
+        exit_code = 0
+        try:
+            run_batch(execute)
+        except SystemExit as error:
+            exit_code = error.code if isinstance(error.code, int) else 1
+            raise
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            cache_hits = self.cache_manager.cache_hit_files
+            _write_run_evidence(
+                evidence_path,
+                project_root=project_root,
+                started_at=started_at,
+                executed_files=[
+                    test_file for test_file in attempted if test_file not in cache_hits
+                ],
+                cache_hits=len(cache_hits),
+                cache_total=len(test_files),
+                no_cache_requested=no_cache,
+                cacheable=blocker is None,
+                cache_blocked_reason=blocker,
+                exit_code=exit_code,
+            )
+
+    def _record_cache_for_result(
+        self,
+        result: subprocess.CompletedProcess,
+        executed_files: list[Path] | None,
+        *,
+        passed_args: list[str],
+        python_executable: str,
+        project_root: str,
+        env: dict[str, str],
+        concise: bool,
+    ) -> None:
+        """Update cache from a batch result, for the files execution was proven.
+
+        A fully-passing suite records those files in bulk; a failing suite is
+        attributed per file (each file re-run through the single-file executor
+        so passing files get cached and failing ones are dropped). A run with
+        no proof records nothing at all (fail-closed, #191).
+        """
+        if result.returncode == 0:
+            if not executed_files:
+                print(
+                    "[Cache] Not saving results: no executed test was proven "
+                    "for this run."
+                )
+            else:
+                self.cache_manager.record_passed_batch(
+                    executed_files,
+                    concise,
+                    python_executable=python_executable,
+                )
+        else:
+            print("Tests failed, updating cache for individual test files...")
+            # The same selection/behavior switches the batch run used, so a
+            # file's verdict is attributed under the filter it was produced
+            # under (a `-k`-excluded failing test must not drop this file).
+            extra_args = _attribution_args(passed_args)
+            scope = executed_files or []
+            if not scope:
+                print(
+                    "[Cache] Skipping per-file attribution: no executed test "
+                    "was proven for this run."
+                )
+            total = len(scope)
+            for index, test_file in enumerate(scope, start=1):
+                # One line per file: this pass is serial and can dwarf the
+                # batch run on a cold cache, so silence is not an option.
+                progress = (
+                    f"[{index}/{total}] "
+                    f"{self._display_test_path(test_file, project_root)}"
+                )
+                log_info_pair(progress, progress)
+                self.single_test_executor.execute(
+                    python_executable,
+                    test_file,
+                    project_root,
+                    env,
+                    debug_mode=False,
+                    no_cache=False,
+                    step_by_step=False,
+                    concise=concise,
+                    extra_args=extra_args,
+                )
+        # Show cache statistics after test run
+        if not concise:
+            self.cache_manager.report_detail()
+        # Cleanup expired cache files (runs in background)
+        try:
+            self.cache_manager.purge_expired()
+        except Exception:
+            # Ignore cleanup errors
+            pass
+
+    def _run_tests_normal(
+        self,
+        python_executable: str,
+        project_root: str,
+        env: dict[str, str],
+        report_mode: bool,
+        passed_args: list[str],
+        debug_mode: bool,
+        todo_dir: Path,
+        no_cache: bool = False,
+        quiet: bool = False,
+        verbose: bool = False,
+        evidence_path: Path | None = None,
+    ) -> None:
+        """Run tests in normal mode with caching.
+
+        When *evidence_path* is given, one structured record is written on
+        **every** terminating path -- the all-cached short circuit above pytest,
+        a passing or failing suite, and an exception -- so a consumer never has
+        to infer "did this run execute anything?" from stdout (#212).
+        """
+        from ...utils.output_context import is_concise_mode
+
+        evidence_started_at = time.time()
+
+        # Start the manager's per-run hit tracking here, as step mode does, so
+        # the evidence's ``cache_hits`` can only ever describe this run (#212
+        # review L1). Normal mode runs one pytest batch and never consults the
+        # cache per file, so this set stays empty today; the record still reads
+        # it instead of writing a literal 0, so a future partial-reuse path
+        # cannot silently report "0 hits" over reused results.
+        self.cache_manager.start_run()
+
+        # Known test files, the subset this run selects, and -- because a green
+        # exit code alone proves nothing (rv191 F1) -- the evidence pytest
+        # writes about what it actually executed.
+        test_files = self._get_all_test_files(project_root)
+        known = {
+            _selector_spelling(test_file, project_root): test_file
+            for test_file in test_files
+        }
+        option_names, _selectors = _split_passthrough_args(passed_args)
+        # Two doors carry pytest switches into the run: the command line and
+        # `PYTEST_ADDOPTS`, which pytest prepends to every command line. Both
+        # are vetted against the same allowlist (#199).
+        cache_blocker = _cache_write_blocker(option_names) or _addopts_cache_blocker(
+            env
+        )
+        selected_files = _executed_test_files(passed_args, project_root, test_files)
+        # The denominator of the "cache usage" fact: the files this run would
+        # consider. An unknown selection still has a known suite size, so the
+        # evidence says "0 of N", not "0 of 0" (#212).
+        cache_total = (
+            len(selected_files) if selected_files is not None else len(test_files)
+        )
+
+        concise = is_concise_mode(override_quiet=quiet, override_verbose=verbose)
+
+        # Check if we can use cached results for every file this run would run.
+        # With an unknown selection (or a run the cache cannot speak about) the
+        # only safe answer is "run, don't cache".
+        all_cached = (
+            not no_cache
+            and cache_blocker is None
+            and selected_files is not None
+            and self.cache_manager.all_cached(
+                selected_files, concise, python_executable=python_executable
+            )
+        )
+
+        if all_cached:
+            # All tests passed in cache, no need to run
+            print("All tests passed successfully! (using cache)")
+            # Show cache statistics
+            self.cache_manager.report_detail()
+            # The short circuit is the very state the gate must judge red, so
+            # its evidence is written explicitly: nothing executed, every file
+            # served from the cache (#212).
+            _write_run_evidence(
+                evidence_path,
+                project_root=project_root,
+                started_at=evidence_started_at,
+                executed_files=[],
+                cache_hits=cache_total,
+                cache_total=cache_total,
+                no_cache_requested=no_cache,
+                cacheable=cache_blocker is None,
+                cache_blocked_reason=cache_blocker,
+                exit_code=0,
+            )
+            return
+
+        # Need to run tests
+        report_path, owns_report = _execution_report(passed_args, report_mode)
+        started_at = time.time()
+        exit_code = 1
+        executed_files: list[Path] | None = None
+        executed_from_report: list[Path] = []
+        try:
+            result = self._run_all_tests(
+                python_executable,
+                project_root,
+                env,
+                report_mode,
+                passed_args,
+                debug_mode,
+                cache_report=report_path,
+            )
+            # pytest's own report is the positive signal behind both the cache
+            # write and the evidence's ``executed_files`` -- derived once, so
+            # the record and the cache scope can never disagree.
+            executed_from_report = _executed_files_from_report(
+                report_path, project_root, known, started_at
+            )
+            executed_files = (
+                None
+                if no_cache
+                else self._cache_write_scope(
+                    cache_blocker, selected_files, executed_from_report
+                )
+            )
+
+            from .test_result_handler import TestResultHandler
+
+            result_handler = TestResultHandler()
+            exit_code = result_handler.handle_test_result(result, report_mode)
+
+            if report_mode:
+                self.report_generator.generate(todo_dir, project_root, verbose)
+
+            if not no_cache:
+                self._record_cache_for_result(
+                    result,
+                    executed_files,
+                    passed_args=passed_args,
+                    python_executable=python_executable,
+                    project_root=project_root,
+                    env=env,
+                    concise=concise,
+                )
+        except SystemExit as error:
+            exit_code = error.code if isinstance(error.code, int) else 1
+            raise
+        except BaseException:
+            if exit_code == 0:  # pragma: no cover - defensive, exit_code starts 1
+                exit_code = 1
+            raise
+        finally:
+            if owns_report and report_path is not None:
+                try:
+                    report_path.unlink()
+                except OSError:
+                    pass
+            _write_run_evidence(
+                evidence_path,
+                project_root=project_root,
+                started_at=evidence_started_at,
+                executed_files=executed_from_report,
+                # The same fact step mode reads, from the manager's own per-run
+                # hit set -- never a literal 0 (#212 review L1).
+                cache_hits=len(self.cache_manager.cache_hit_files),
+                cache_total=cache_total,
+                no_cache_requested=no_cache,
+                cacheable=cache_blocker is None,
+                cache_blocked_reason=cache_blocker,
+                exit_code=exit_code,
+            )
+
+        # Exit only after report generation and cache attribution have run,
+        # so a failing suite still produces its reports and cache state.
+        if exit_code:
+            sys.exit(exit_code)