flakehound 0.1.0__py3-none-any.whl

flakehound/adapters/pytest/runner.py

@@ -0,0 +1,208 @@
+"""Runs pytest in subprocesses, one process per experiment.
+
+Why a fresh process per run: every run starts from a clean interpreter, so state
+can only leak *within* a run (which is what we want to observe), a crashing or
+hanging test can't take flakehound down, and cleanup is reliable.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import time
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+from xml.etree import ElementTree
+
+from flakehound.adapters.base import CollectionError
+from flakehound.core.models import Repro, RunResult, RunSpec, StateChange, TestResult
+from flakehound.core.procs import ProcessRegistry
+from flakehound.report.redact import redact, truncate
+
+PLUGIN = "flakehound.adapters.pytest.plugin"
+
+# Plugins that reorder tests would fight our ordering, and quarantine would hide
+# the very failures we are hunting.
+_BLOCKED_PLUGINS = ("randomly", "random_order", "cacheprovider", "flakehound-quarantine")
+
+
+class PytestRunner:
+    name = "pytest"
+
+    def __init__(
+        self,
+        project_dir: Path,
+        pytest_args: Sequence[str] = (),
+        *,
+        python: str | None = None,
+        procs: ProcessRegistry,
+        timeout: float | None = 600.0,
+        base_env: Mapping[str, str] | None = None,
+    ) -> None:
+        self.project_dir = project_dir
+        self.pytest_args = list(pytest_args)
+        self.python = python or sys.executable
+        self.procs = procs
+        self.timeout = timeout
+        self.base_env = dict(os.environ if base_env is None else base_env)
+        self._xdist: bool | None = None
+
+    @property
+    def has_xdist(self) -> bool:
+        """Is pytest-xdist installed for the test interpreter? (checked once)
+
+        If so we pass ``-n 0``: a ``-n auto`` in the project's addopts would
+        otherwise spread tests over processes and break the exact orders we
+        run. (``-p no:xdist`` would make such addopts an error instead.)
+        """
+        if self._xdist is None:
+            probe = (
+                "import importlib.util, sys; "
+                "sys.exit(0 if importlib.util.find_spec('xdist') else 1)"
+            )
+            res = self.procs.run(
+                [self.python, "-c", probe], cwd=self.project_dir, env=self._env(), timeout=60
+            )
+            self._xdist = res.returncode == 0 and not res.timed_out
+        return self._xdist
+
+    def _command(self, *extra: str) -> list[str]:
+        cmd = [self.python, "-m", "pytest", "-p", PLUGIN]
+        for plugin in _BLOCKED_PLUGINS:
+            cmd += ["-p", f"no:{plugin}"]
+        cmd += ["-q", "--no-header", "-o", "console_output_style=classic"]
+        if self.has_xdist:
+            cmd += ["-n", "0"]
+        cmd += list(extra)
+        cmd += self.pytest_args
+        return cmd
+
+    def _env(self, spec: RunSpec | None = None) -> dict[str, str]:
+        env = dict(self.base_env)
+        env["PYTHONDONTWRITEBYTECODE"] = "1"
+        if spec is not None:
+            if spec.hash_seed is not None:
+                env["PYTHONHASHSEED"] = str(spec.hash_seed)
+            env.update(spec.env)
+        return env
+
+    def collect(self, workdir: Path) -> list[str]:
+        out = workdir / "collected.json"
+        res = self.procs.run(
+            self._command("--collect-only", f"--flakehound-collect-out={out}"),
+            cwd=self.project_dir,
+            env=self._env(),
+            timeout=self.timeout,
+        )
+        if res.timed_out:
+            raise CollectionError("pytest collection timed out")
+        # Exit code 5 = no tests collected, which is not an error for us.
+        if res.returncode not in (0, 5) or not out.exists():
+            detail = truncate(redact((res.stdout + "\n" + res.stderr).strip()), 4000)
+            raise CollectionError(f"pytest could not collect tests:\n{detail}")
+        ids = json.loads(out.read_text(encoding="utf-8"))
+        return [str(x) for x in ids]
+
+    def run(self, spec: RunSpec, workdir: Path) -> RunResult:
+        spec_path = workdir / f"spec-{spec.run_id}.json"
+        out_path = workdir / f"result-{spec.run_id}.json"
+        spec_path.write_text(json.dumps(spec.to_json()), encoding="utf-8")
+        start = time.perf_counter()
+        res = self.procs.run(
+            self._command(f"--flakehound-spec={spec_path}", f"--flakehound-out={out_path}"),
+            cwd=self.project_dir,
+            env=self._env(spec),
+            timeout=self.timeout,
+        )
+        duration = time.perf_counter() - start
+        results: list[TestResult] = []
+        state_changes: dict[str, list[StateChange]] = {}
+        sleep_sites: dict[str, list[dict[str, Any]]] = {}
+        if out_path.exists():
+            data = json.loads(out_path.read_text(encoding="utf-8"))
+            results = [TestResult.from_json(r) for r in data["results"]]
+            state_changes = {
+                str(nodeid): [StateChange.from_json(c) for c in changes]
+                for nodeid, changes in data.get("state_changes", {}).items()
+            }
+            sleep_sites = {str(k): list(v) for k, v in data.get("sleep_sites", {}).items()}
+        crashed = res.timed_out or not out_path.exists()
+        error = ""
+        if res.timed_out:
+            error = f"run timed out after {self.timeout:.0f}s"
+        elif crashed:
+            tail = "\n".join((res.stdout + "\n" + res.stderr).strip().splitlines()[-20:])
+            error = truncate(redact(f"worker exited with code {res.returncode}:\n{tail}"))
+        return RunResult(
+            spec, results, res.returncode, duration, crashed, error, state_changes, sleep_sites
+        )
+
+    def repro_command(self, repro: Repro) -> list[str]:
+        """A pytest command that reproduces a finding (env vars are listed separately).
+
+        pytest runs explicitly listed test ids in the order they are given.
+        Seed and clock replay need flakehound installed (they use its plugin).
+        """
+        cmd = ["pytest", "-p", "no:randomly", "-p", "no:cacheprovider"]
+        if self.has_xdist:
+            cmd += ["-n", "0"]  # keep the exact order even if addopts has -n auto
+        flags = []
+        if repro.seed is not None:
+            flags.append(f"--flakehound-seed={repro.seed}")
+        if repro.clock is not None:
+            flags.append(f"--flakehound-clock={repro.clock}")
+        if repro.sleep_scale is not None:
+            flags.append(f"--flakehound-sleep-scale={repro.sleep_scale:g}")
+        if repro.sleep_sites:
+            flags.append("--flakehound-sleep-sites=" + ",".join(repro.sleep_sites))
+        if repro.switch_interval is not None:
+            flags.append(f"--flakehound-switch-interval={repro.switch_interval:g}")
+        if flags:
+            cmd += ["-p", PLUGIN, *flags]
+        return [*cmd, *repro.order]
+
+    def check_repro(self, repro: Repro) -> bool:
+        """Run :meth:`repro_command` for real; True if the last test failed.
+
+        This proves the exact command we print reproduces the failure. The last
+        test's outcome is read from a JUnit XML report (part of pytest itself).
+        """
+        cmd = [self.python, "-m", *self.repro_command(repro)]
+        with tempfile.TemporaryDirectory(prefix="flakehound-repro-") as tmp:
+            xml = Path(tmp) / "repro.xml"
+            run_env = dict(self.base_env)
+            run_env["PYTHONDONTWRITEBYTECODE"] = "1"
+            run_env.update(repro.env)
+            res = self.procs.run(
+                [*cmd, "-q", f"--junitxml={xml}"],
+                cwd=self.project_dir,
+                env=run_env,
+                timeout=self.timeout,
+            )
+            if res.timed_out or not xml.exists():
+                return False
+            return _last_test_failed(xml, repro.order[-1])
+
+
+def _last_test_failed(xml_path: Path, nodeid: str) -> bool:
+    """Did ``nodeid`` fail in a pytest JUnit XML report?"""
+    # The XML is written by pytest from the user's own project, parsed with the
+    # stdlib parser, which does not resolve external entities.
+    tree = ElementTree.parse(xml_path)  # noqa: S314
+    parts = nodeid.split("::")
+    name = parts[-1]
+    # JUnit classname is the dotted file path plus any test classes,
+    # e.g. "tests/test_a.py::TestX::test_y" -> "tests.test_a.TestX".
+    module = parts[0].removesuffix(".py").replace("/", ".").replace("\\", ".")
+    classname = ".".join([module, *parts[1:-1]])
+    for case in tree.iter("testcase"):
+        if (
+            case.get("name") == name
+            and case.get("classname") == classname
+            and (case.find("failure") is not None or case.find("error") is not None)
+        ):
+            return True
+    return False

flakehound/adapters/pytest/snapshot.py

@@ -0,0 +1,205 @@
+"""Snapshot and diff process-global state around one test.
+
+Used to name *what* a polluter test leaked: "test_add_item leaves
+shop.cart = ['apple']" is far more useful than "test_add_item leaks state".
+
+What we look at (cheap, read-only):
+* module-level variables of the project's own modules (not site-packages),
+  one level into plain objects and class attributes;
+* ``os.environ``, the current directory, ``sys.path``;
+* files in the project directory (name, size, mtime);
+* the root logger's level and handlers, and ``logging.disable``.
+
+Limits, stated honestly: state hidden in fixtures, closures, C extensions,
+databases or external services is invisible here.
+
+Every value is passed through :func:`redact` and truncated, and environment
+variables with secret-sounding names never show their values.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import reprlib
+import sys
+import types
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from flakehound.adapters.pytest.paths import canon, is_project_file
+from flakehound.core.models import StateChange
+from flakehound.report.redact import REDACTED, redact
+
+MISSING = "<missing>"
+_MAX_VALUE = 120
+_MAX_ATTRS = 50
+_MAX_FILES = 20_000
+_MAX_CHANGES = 25
+_SKIP_DIRS = {
+    ".git",
+    ".hg",
+    ".svn",
+    ".venv",
+    "venv",
+    "env",
+    ".env",
+    "node_modules",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".tox",
+    ".nox",
+    "build",
+    "dist",
+    ".eggs",
+}
+_SKIP_MODULE_PREFIXES = ("flakehound", "_pytest", "pytest", "pluggy", "py.")
+# Set and cleared by pytest itself around every test: never "leaked state".
+_IGNORED_ENV = {"PYTEST_CURRENT_TEST"}
+_SECRET_ENV_NAME = re.compile(r"(?i)(pass|secret|token|key|auth|credential|cookie|session)")
+
+_repr = reprlib.Repr()
+_repr.maxstring = 60
+_repr.maxother = 60
+_repr.maxlist = _repr.maxtuple = _repr.maxset = _repr.maxfrozenset = _repr.maxdeque = 8
+_repr.maxdict = 8
+_repr.maxlevel = 3
+
+
+def safe_repr(value: object) -> str:
+    try:
+        text = _repr.repr(value)
+    except Exception:  # a user __repr__ may raise anything
+        text = f"<{type(value).__name__} (repr failed)>"
+    try:
+        size = len(value)  # type: ignore[arg-type]
+    except Exception:
+        size = None
+    # Truncated containers can hide in-place changes: include the length.
+    if size is not None and isinstance(value, (list, dict, set, tuple, frozenset)):
+        text = f"{text} (len {size})" if size > _repr.maxlist else text
+    return redact(text if len(text) <= _MAX_VALUE else text[: _MAX_VALUE - 3] + "...")
+
+
+@dataclass
+class Snapshot:
+    values: dict[tuple[str, str], str] = field(default_factory=dict)  # (kind, name) -> repr
+    files: dict[str, tuple[int, int]] = field(default_factory=dict)  # path -> (size, mtime)
+
+
+def _is_project_module(name: str, module: types.ModuleType, root: str) -> bool:
+    if name.startswith(_SKIP_MODULE_PREFIXES) or name == "__main__":
+        return False
+    file = getattr(module, "__file__", None)
+    return bool(file) and is_project_file(str(file), root)
+
+
+def _plain_attrs(obj: object) -> Iterator[tuple[str, object]]:
+    try:
+        attrs = vars(obj)
+    except TypeError:
+        return
+    for i, (key, value) in enumerate(list(attrs.items())):
+        if i >= _MAX_ATTRS:
+            break
+        if key.startswith("__") or callable(value) or isinstance(value, types.ModuleType):
+            continue
+        yield key, value
+
+
+def _module_state(root: Path) -> Iterator[tuple[str, str, str]]:
+    root_s = canon(str(root))
+    for mod_name, module in list(sys.modules.items()):
+        if module is None or not _is_project_module(mod_name, module, root_s):
+            continue
+        for attr, value in list(vars(module).items()):
+            if attr.startswith("__") or isinstance(value, types.ModuleType):
+                continue
+            if isinstance(value, type):
+                if getattr(value, "__module__", None) != mod_name:
+                    continue  # imported class: owned by another module
+                for key, cval in _plain_attrs(value):
+                    yield "class_attr", f"{mod_name}.{attr}.{key}", safe_repr(cval)
+                continue
+            if callable(value):
+                continue  # functions, and objects like mocks are caught via repr below
+            yield "global", f"{mod_name}.{attr}", safe_repr(value)
+            # One level into plain objects: catches app.state.CURRENCY = "EUR".
+            if not isinstance(value, (str, bytes, int, float, bool, list, dict, set, tuple)):
+                for key, aval in _plain_attrs(value):
+                    yield "global", f"{mod_name}.{attr}.{key}", safe_repr(aval)
+
+
+def _files(root: Path) -> dict[str, tuple[int, int]]:
+    out: dict[str, tuple[int, int]] = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS and not d.endswith(".egg-info")]
+        for name in filenames:
+            if name.endswith((".pyc", ".pyo")):
+                continue
+            path = Path(dirpath, name)
+            try:
+                st = path.stat()
+            except OSError:
+                continue
+            out[path.relative_to(root).as_posix()] = (st.st_size, st.st_mtime_ns)
+            if len(out) >= _MAX_FILES:
+                return out
+    return out
+
+
+def _env_value(name: str, value: str) -> str:
+    if _SECRET_ENV_NAME.search(name):
+        return REDACTED
+    return safe_repr(value)
+
+
+def take(root: Path) -> Snapshot:
+    snap = Snapshot()
+    v = snap.values
+    for kind, name, rep in _module_state(root):
+        v[(kind, name)] = rep
+    for key, value in os.environ.items():
+        if key not in _IGNORED_ENV:
+            v[("env", key)] = _env_value(key, value)
+    try:
+        v[("cwd", "current directory")] = redact(os.getcwd())
+    except OSError:
+        v[("cwd", "current directory")] = "<deleted>"
+    v[("sys_path", "sys.path")] = safe_repr(list(sys.path))
+    root_logger = logging.getLogger()
+    v[("logging", "root logger level")] = logging.getLevelName(root_logger.level)
+    v[("logging", "root logger handlers")] = safe_repr(
+        [type(h).__name__ for h in root_logger.handlers]
+    )
+    v[("logging", "logging.disable")] = str(logging.root.manager.disable)
+    snap.files = _files(root)
+    return snap
+
+
+def diff(before: Snapshot, after: Snapshot) -> list[StateChange]:
+    changes: list[StateChange] = []
+    for key in sorted(set(before.values) | set(after.values)):
+        old = before.values.get(key, MISSING)
+        new = after.values.get(key, MISSING)
+        if old != new:
+            changes.append(StateChange(key[0], key[1], old, new))
+    for path in sorted(set(before.files) | set(after.files)):
+        old_f = before.files.get(path)
+        new_f = after.files.get(path)
+        if old_f == new_f:
+            continue
+        if old_f is None:
+            changes.append(StateChange("file", path, MISSING, "created"))
+        elif new_f is None:
+            changes.append(StateChange("file", path, "existed", "deleted"))
+        else:
+            changes.append(StateChange("file", path, "existed", "modified"))
+    # Most useful first: globals, then env, then the rest.
+    rank = {"global": 0, "class_attr": 1, "env": 2, "cwd": 3, "file": 4, "sys_path": 5}
+    changes.sort(key=lambda c: (rank.get(c.kind, 9), c.name))
+    return changes[:_MAX_CHANGES]

flakehound/adapters/pytest/timing.py

@@ -0,0 +1,141 @@
+"""Timing stress inside a worker: shorter waits and frequent thread switches.
+
+* **Sleep shrinking.** ``time.sleep`` is wrapped. Sleeps made on the *main
+  thread* (where the test waits) can be scaled down, either all of them or only
+  the ones at given call sites (``path:line``). Sleeps in background threads
+  (which usually simulate work) are left alone. If a test fails when its wait
+  is shortened, it was using ``sleep`` to wait for something that is not
+  guaranteed to be done: on a slower machine it will fail the same way.
+* **Call-site recording.** Every main-thread sleep is attributed to the first
+  stack frame in the project's own code, so we can say *which* sleep matters.
+* **Thread-switch stress.** ``sys.setswitchinterval`` makes the interpreter
+  switch threads far more often, and a profile hook briefly yields the GIL
+  every time any thread enters a function of the project's own code. The
+  yield matters on systems (like macOS) where a waiting thread wakes up too
+  slowly for the switch interval alone to interleave short critical sections.
+  Together they expose races that are rare by default.
+
+``time.monotonic``, ``perf_counter`` and ``asyncio.sleep`` are not touched.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import time as _time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from types import FrameType
+
+from flakehound.adapters.pytest.paths import canon, is_project_file, real
+
+_real_sleep: Callable[[float], None] = _time.sleep
+
+
+@dataclass
+class SleepSite:
+    site: str  # "tests/test_upload.py:9"
+    seconds: float  # longest requested sleep at this site
+    count: int = 0
+
+    def to_json(self) -> dict[str, object]:
+        return {"site": self.site, "seconds": round(self.seconds, 6), "count": self.count}
+
+
+@dataclass
+class _State:
+    root: str = ""
+    scale: float | None = None
+    only_sites: set[str] | None = None
+    current_test: str | None = None
+    sites: dict[str, dict[str, SleepSite]] = field(default_factory=dict)  # test -> site -> info
+    lock: threading.Lock = field(default_factory=threading.Lock)
+
+
+_state = _State()
+_root_real = ""  # resolved root, original case (for display)
+
+
+def _project_site(frame: FrameType | None) -> str | None:
+    """``relative/path.py:line`` of the first frame in the project's own code."""
+    while frame is not None:
+        file = frame.f_code.co_filename
+        if is_project_file(file, _state.root):
+            rel = os.path.relpath(real(file), _root_real)
+            return f"{rel.replace(os.sep, '/')}:{frame.f_lineno}"
+        frame = frame.f_back
+    return None
+
+
+def _sleep(seconds: float) -> None:
+    if threading.current_thread() is threading.main_thread() and _state.current_test:
+        site = _project_site(sys._getframe(1))
+        if site is not None:
+            with _state.lock:
+                per_test = _state.sites.setdefault(_state.current_test, {})
+                info = per_test.setdefault(site, SleepSite(site, 0.0))
+                info.count += 1
+                info.seconds = max(info.seconds, float(seconds))
+            if _state.scale is not None and (
+                _state.only_sites is None or site in _state.only_sites
+            ):
+                seconds = seconds * _state.scale
+    _real_sleep(seconds)
+
+
+def install(
+    root: str,
+    *,
+    scale: float | None = None,
+    only_sites: list[str] | None = None,
+    switch_interval: float | None = None,
+) -> None:
+    """Patch ``time.sleep`` (recording always; shrinking if ``scale``)."""
+    global _root_real
+    _state.root = canon(root)
+    _root_real = real(root)
+    _state.scale = scale
+    _state.only_sites = set(only_sites) if only_sites else None
+    if _time.sleep is not _sleep:
+        _time.sleep = _sleep  # type: ignore[assignment]
+        _rebind_project_modules()
+    if switch_interval is not None:
+        sys.setswitchinterval(switch_interval)
+        threading.setprofile(_yield_on_call)
+        sys.setprofile(_yield_on_call)
+
+
+def _rebind_project_modules() -> None:
+    """``from time import sleep`` in already-imported project modules (e.g. conftest)."""
+    for module in list(sys.modules.values()):
+        file = getattr(module, "__file__", None)
+        if not file or not is_project_file(file, _state.root):
+            continue
+        for name, value in list(vars(module).items()):
+            if value is _real_sleep:
+                setattr(module, name, _sleep)
+
+
+_project_code: dict[str, bool] = {}
+
+
+def _yield_on_call(frame: FrameType, event: str, arg: object) -> None:
+    """Profile hook: let other threads run whenever project code calls a function."""
+    if event != "call":
+        return
+    file = frame.f_code.co_filename
+    mine = _project_code.get(file)
+    if mine is None:
+        mine = _project_code[file] = is_project_file(file, _state.root)
+    if mine:
+        _real_sleep(1e-6)  # releases the GIL; a waiting thread can take over
+
+
+def set_current_test(nodeid: str | None) -> None:
+    _state.current_test = nodeid
+
+
+def sites_for(nodeid: str) -> list[SleepSite]:
+    with _state.lock:
+        return sorted(_state.sites.get(nodeid, {}).values(), key=lambda s: s.site)