workflowval 0.1.0__py3-none-any.whl

workflowval/gen/__init__.py

@@ -0,0 +1,194 @@
+"""Renders pytest files from workflow files, with no model involved (FR-5)."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from workflowval.gen.source import SourceError, check_test_source, format_source, lint_source
+from workflowval.schemas import AuthConfig, LoginAuth, Workflow
+from workflowval.schemas.workflow import _PLACEHOLDER, CleanupStep, KnownIssue, Step
+from workflowval.spec import SpecGraph
+
+HASH_PREFIX = "# workflow-sha256: "
+
+
+class GenerationError(Exception):
+    """A workflow can't be rendered against this spec."""
+
+
+@dataclass(frozen=True)
+class TestFile:
+    """A generated test file."""
+
+    __test__ = False
+
+    path: str
+    source: str
+
+
+def workflow_hash(workflow: Workflow) -> str:
+    """Hash a workflow, so a test is regenerated only when its workflow changed."""
+    return hashlib.sha256(workflow.model_dump_json().encode()).hexdigest()
+
+
+def hash_line(workflow: Workflow) -> str:
+    """The first line of a generated test, tying it to its workflow."""
+    return f"{HASH_PREFIX}{workflow_hash(workflow)}"
+
+
+def test_path(workflow_id: str) -> str:
+    """Where a workflow's test file lives."""
+    return f"tests/wf/test_{workflow_id}.py"
+
+
+def _expr(value: Any) -> str:
+    """Render a JSON value as Python. Text goes through repr, so it can never become code."""
+    if isinstance(value, str):
+        match = _PLACEHOLDER.fullmatch(value)
+        if match:
+            name = match.group(1)
+            return f"api.{name}" if name in {"unique", "unique_int"} else f"saved[{name!r}]"
+        return f"api.fill({value!r}, saved)" if _PLACEHOLDER.search(value) else repr(value)
+    if isinstance(value, dict):
+        items = ", ".join(f"{key!r}: {_expr(item)}" for key, item in value.items())
+        return "{" + items + "}"
+    if isinstance(value, list):
+        return "[" + ", ".join(_expr(item) for item in value) + "]"
+    return repr(value)
+
+
+def _call_arguments(step: CleanupStep, graph: SpecGraph) -> str:
+    operation = graph.operation(step.op)
+    if operation is None:
+        raise GenerationError(f"operation {step.op!r} isn't in the spec")
+    groups: dict[str, dict[str, Any]] = {"path": {}, "params": {}, "headers": {}}
+    target = {"path": "path", "query": "params", "header": "headers"}
+    locations = {parameter.name: parameter.location for parameter in operation.parameters}
+    for name, value in step.params.items():
+        location = locations.get(name)
+        if location not in target:
+            raise GenerationError(f"{step.op} has no path, query or header parameter {name!r}")
+        groups[target[location]][name] = value
+    parts = [repr(operation.method), repr(operation.path)]
+    parts += [f"{key}={_expr(group)}" for key, group in groups.items() if group]
+    if step.body is not None:
+        parts.append(f"body={_expr(step.body)}")
+    if isinstance(step, Step) and not step.auth:
+        parts.append("auth=False")
+    return ", ".join(parts)
+
+
+def _render_step(step: Step, graph: SpecGraph, issue: KnownIssue | None = None) -> list[str]:
+    """Render one step. `issue` is set only on the step with a reviewed, expected failure."""
+    lines = [f"response = api.call({_call_arguments(step, graph)})"]
+    if issue is not None and issue.check == "status":
+        lines += [f"if response.status_code == {issue.status}:", "    api.expected_failure()"]
+    lines.append(f"assert response.status_code == {step.expect.status}, response.text")
+    checks = [f"saved[{k!r}] = api.value(response, {v!r})" for k, v in step.save.items()]
+    for path, value in step.expect.body.items():
+        # True, False and None compare by identity, so a truthy 1 can't pass as True.
+        operator = "is" if value is None or isinstance(value, bool) else "=="
+        checks.append(f"assert api.value(response, {path!r}) {operator} {_expr(value)}")
+    if issue is not None and issue.check == "body" and checks:
+        lines.append("with api.expected_failure_on_assert():")
+        return lines + [f"    {line}" for line in checks]
+    return lines + checks
+
+
+def _render_cleanup(step: CleanupStep, graph: SpecGraph) -> list[str]:
+    used = sorted(
+        set(_PLACEHOLDER.findall(repr([step.params, step.body]))) - {"unique", "unique_int"}
+    )
+    call = f"api.cleanup({_call_arguments(step, graph)})"
+    if not used:
+        return [call]
+    guard = " and ".join(f"{name!r} in saved" for name in used)
+    return [f"if {guard}:", f"    {call}"]
+
+
+def generate(workflow: Workflow, graph: SpecGraph) -> TestFile:
+    """Render a workflow as one readable pytest file."""
+    issue = workflow.known_issue
+    body = [
+        line
+        for index, step in enumerate(workflow.steps)
+        for line in _render_step(step, graph, issue if issue and issue.step == index else None)
+    ]
+    cleanup = [line for step in workflow.cleanup for line in _render_cleanup(step, graph)]
+    lines = [hash_line(workflow), repr(workflow.rule), "", "", f"def test_{workflow.id}(api):"]
+    if issue is not None:
+        lines.append(f"    api.known_issue({f'{issue.label}: {issue.reason}'!r})")
+    if "saved" in "\n".join([*body, *cleanup]):
+        lines.append("    saved = {}")
+    if cleanup:
+        lines.append("    try:")
+        lines += [f"        {line}" for line in body]
+        lines.append("    finally:")
+        lines += [f"        {line}" for line in cleanup]
+    else:
+        lines += [f"    {line}" for line in body]
+    try:
+        source = format_source("\n".join(lines) + "\n")
+        lint_source(source)
+    except SourceError as exc:
+        raise GenerationError(str(exc)) from exc
+    return TestFile(test_path(workflow.id), source)
+
+
+def token_style(scheme: LoginAuth, graph: SpecGraph) -> tuple[str, str]:
+    """Decide how to send a login token: config first, then the spec's security section."""
+    header, prefix = scheme.token_header, scheme.token_prefix
+    for security in graph.security_schemes.values():
+        if header is not None and prefix is not None:
+            break
+        kind = security.get("type")
+        if kind == "http" and str(security.get("scheme", "")).lower() == "bearer":
+            header, prefix = header or "Authorization", "Bearer " if prefix is None else prefix
+        elif kind == "apiKey" and security.get("in") == "header":
+            header, prefix = header or security["name"], "" if prefix is None else prefix
+        elif kind == "apiKey" and security.get("in") == "cookie":
+            default_prefix = f"{security['name']}="
+            header, prefix = header or "Cookie", default_prefix if prefix is None else prefix
+    return header or "Authorization", "Bearer " if prefix is None else prefix
+
+
+def _auth_settings(auth: AuthConfig | None, graph: SpecGraph) -> dict[str, Any] | None:
+    if auth is None:
+        return None
+    scheme = auth.scheme
+    settings: dict[str, Any] = scheme.model_dump()
+    if isinstance(scheme, LoginAuth):
+        operation = graph.operation(scheme.op)
+        if operation is None:
+            raise GenerationError(f"auth config: operation {scheme.op!r} isn't in the spec")
+        settings["path"], settings["method"] = operation.path, operation.method
+        settings["token_header"], settings["token_prefix"] = token_style(scheme, graph)
+    return settings
+
+
+def render_conftest(base_url: str, auth: AuthConfig | None, graph: SpecGraph) -> str:
+    """Render the shared conftest.py. It holds env var names only, never secret values."""
+    template = resources.files(__package__).joinpath("templates/conftest.py.tmpl")
+    source = template.read_text(encoding="utf-8")
+    source = source.replace("__BASE_URL__", repr(base_url))
+    source = source.replace("__AUTH__", repr(_auth_settings(auth, graph)))
+    return format_source(source)
+
+
+__all__ = [
+    "GenerationError",
+    "SourceError",
+    "TestFile",
+    "check_test_source",
+    "format_source",
+    "generate",
+    "hash_line",
+    "lint_source",
+    "render_conftest",
+    "test_path",
+    "token_style",
+    "workflow_hash",
+]

workflowval/gen/source.py

@@ -0,0 +1,109 @@
+"""Formats, lints and checks generated test code. Everything here runs before code is executed."""
+
+from __future__ import annotations
+
+import ast
+import subprocess
+import sys
+
+from workflowval.schemas import Workflow
+
+ALLOWED_IMPORTS = frozenset({"pytest", "httpx", "json", "re", "uuid", "time", "datetime"})
+FORBIDDEN_NAMES = frozenset(
+    {"eval", "exec", "compile", "open", "__import__", "globals", "locals", "getattr", "setattr"}
+)
+FORBIDDEN_ATTRIBUTES = frozenset({"skip", "skipif", "xfail", "importorskip"})
+EXPECTED_FAILURE_CALLS = frozenset(
+    {"known_issue", "expected_failure", "expected_failure_on_assert"}
+)
+_RUFF_ARGS = ("--isolated", "--line-length", "100", "--stdin-filename", "test_generated.py")
+
+
+class SourceError(Exception):
+    """Test code was refused. The message says why."""
+
+
+def _ruff(*args: str, source: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(  # noqa: S603 - fixed arguments, and source goes in through stdin
+        [sys.executable, "-m", "ruff", *args],
+        input=source,
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=60,
+    )
+
+
+def format_source(source: str) -> str:
+    """Format Python source with ruff."""
+    result = _ruff("format", *_RUFF_ARGS, "-", source=source)
+    if result.returncode != 0:
+        raise SourceError(f"code can't be formatted: {result.stderr.strip()}")
+    return result.stdout
+
+
+def lint_source(source: str) -> None:
+    """Refuse source that doesn't pass ruff."""
+    result = _ruff(
+        "check",
+        *_RUFF_ARGS,
+        "--select",
+        "E,F,I,B",
+        "--ignore",
+        "E501",
+        "--output-format",
+        "concise",
+        "-",
+        source=source,
+    )
+    if result.returncode != 0:
+        raise SourceError(f"ruff found problems: {result.stdout.strip()}")
+
+
+def count_assertions(tree: ast.AST) -> int:
+    """Count the assert statements in parsed source."""
+    return sum(isinstance(node, ast.Assert) for node in ast.walk(tree))
+
+
+def check_test_source(source: str, workflow: Workflow, previous: str | None = None) -> None:
+    """Refuse an edited test that could run something unexpected or hide a real failure.
+
+    This is a safety check on model-written code, not a sandbox.
+    """
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as exc:
+        raise SourceError(f"not valid Python: {exc}") from exc
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            modules = [alias.name for alias in node.names]
+        elif isinstance(node, ast.ImportFrom):
+            modules = [node.module or "."]
+        else:
+            modules = []
+        for module in modules:
+            if module.split(".")[0] not in ALLOWED_IMPORTS:
+                raise SourceError(f"import of {module!r} isn't allowed in a generated test")
+        if isinstance(node, ast.Name) and node.id in FORBIDDEN_NAMES:
+            raise SourceError(f"{node.id!r} isn't allowed in a generated test")
+        if isinstance(node, ast.Attribute) and node.attr in FORBIDDEN_ATTRIBUTES:
+            raise SourceError(f"pytest {node.attr} isn't allowed: a real failure must stay visible")
+        if (
+            isinstance(node, ast.Attribute)
+            and node.attr in EXPECTED_FAILURE_CALLS
+            and workflow.known_issue is None
+        ):
+            raise SourceError(f"{node.attr} isn't allowed: only a reviewed failure can be expected")
+    functions = [node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)]
+    names = [function.name for function in functions if function.name.startswith("test")]
+    expected = f"test_{workflow.id}"
+    if names != [expected]:
+        raise SourceError(f"the file must define exactly one test, {expected}; found {names}")
+    constants = {node.value for node in ast.walk(tree) if isinstance(node, ast.Constant)}
+    for step in workflow.steps:
+        if step.expect.status not in constants:
+            raise SourceError(f"the test no longer checks status {step.expect.status} ({step.op})")
+    if previous is not None:
+        before, after = count_assertions(ast.parse(previous)), count_assertions(tree)
+        if after < before:
+            raise SourceError(f"assertion weakening refused: {before} assertions became {after}")

workflowval/gen/style.md

@@ -0,0 +1,17 @@
+# Generated test style
+
+Tests are rendered from workflow files by `workflowval.gen`, so these rules hold by construction.
+The Writer model only edits a test to fix a failure labeled `TEST_BUG`, and `write_test` enforces
+the rules below on its edits.
+
+- One file per workflow, `tests/wf/test_<id>.py`, with one function, `test_<id>(api)`.
+- Plain pytest + httpx through the `api` fixture in `conftest.py`. No workflowval import.
+- Each step's status code and values are checked with a plain `assert`.
+- Cleanup runs in `finally`, and only touches data the test created.
+- Fresh data per run comes from `api.unique` and `api.unique_int`.
+- No secrets in test files: credentials come from env vars inside `conftest.py`.
+- Only these imports: pytest, httpx, json, re, uuid, time, datetime.
+- No `skip` or `xfail`, and no fewer assertions than before. A real failure must stay visible.
+- Only code may mark a reviewed failure as expected, through `known_issue` in the workflow file. It
+  excuses that one check on that one step, and any other failure still fails the test.
+- `ruff` passes (rules E, F, I, B; line length isn't enforced on long literals).

workflowval/gen/templates/conftest.py.tmpl

@@ -0,0 +1,332 @@
+"""Generated by workflowval. Shared test helpers; needs only pytest and httpx to run."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import secrets
+import uuid
+from collections.abc import Generator, Iterator
+from contextlib import contextmanager
+from typing import Any
+from urllib.parse import quote
+
+import httpx
+import pytest
+
+BASE_URL = os.environ.get("WORKFLOWVAL_BASE_URL", __BASE_URL__).rstrip("/")
+AUTH: dict[str, Any] | None = __AUTH__
+TIMEOUT_SECONDS = 30
+READ_ONLY = os.environ.get("WORKFLOWVAL_READ_ONLY") == "1"
+KEEP_RAW_BODIES = os.environ.get("WORKFLOWVAL_NO_REDACT") == "1"
+ALLOWED_HOSTS = {
+    httpx.URL(BASE_URL).host,
+    *filter(None, os.environ.get("WORKFLOWVAL_ALLOWED_HOSTS", "").split(",")),
+}
+SAFE_METHODS = {"GET", "HEAD", "OPTIONS"}
+MASK = "[REDACTED]"
+SECRET_HEADERS = {"authorization", "cookie", "set-cookie", "proxy-authorization", "x-api-key"}
+if AUTH and AUTH.get("type") == "api_key" and AUTH["location"] == "header":
+    SECRET_HEADERS.add(AUTH["name"].lower())
+
+_secrets: set[str] = set()
+_exchanges: list[httpx.Response] = []
+_MAX_RAW_BODY = 200_000
+_recording = True
+_results: dict[str, dict[str, Any]] = {}
+_steps_run = 0
+_known_issue: str | None = None
+_PATH_PART = re.compile(r"\.([A-Za-z_][\w-]*)|\[(\d+)\]")
+_PLACEHOLDER = re.compile(r"\{([A-Za-z_]\w*)\}")
+
+
+def _env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise RuntimeError(f"environment variable {name} is not set")
+    _secrets.add(value)
+    return value
+
+
+def _mask(text: str) -> str:
+    for value in sorted(_secrets, key=len, reverse=True):
+        text = text.replace(value, MASK)
+    if AUTH and AUTH.get("type") == "api_key" and AUTH["location"] == "query":
+        text = re.sub(rf"({re.escape(AUTH['name'])}=)[^&\s]+", rf"\g<1>{MASK}", text)
+    return text
+
+
+def _value_at(document: Any, path: str) -> Any:
+    node = document
+    for name, index in _PATH_PART.findall(path):
+        if name:
+            if not isinstance(node, dict) or name not in node:
+                raise AssertionError(f"no value at {path} in the response")
+            node = node[name]
+        else:
+            if not isinstance(node, list) or int(index) >= len(node):
+                raise AssertionError(f"no value at {path} in the response")
+            node = node[int(index)]
+    return node
+
+
+def _nested(fields: dict[str, str]) -> dict[str, Any]:
+    """Turn `user.email` into {"user": {"email": ...}}, for APIs that nest the login body."""
+    body: dict[str, Any] = {}
+    for name, value in fields.items():
+        *parents, last = name.split(".")
+        node = body
+        for parent in parents:
+            node = node.setdefault(parent, {})
+        node[last] = value
+    return body
+
+
+class _LoginAuth(httpx.Auth):
+    """Logs in once, reuses the token and logs in again on a 401."""
+
+    requires_response_body = True
+
+    def __init__(self, scheme: dict[str, Any]) -> None:
+        self.scheme = scheme
+        self.token: str | None = None
+
+    def _login(self) -> Generator[httpx.Request, httpx.Response, None]:
+        scheme = self.scheme
+        fields = {name: _env(env) for name, env in scheme["body_env"].items()}
+        url = BASE_URL + scheme["path"]
+        if scheme["method"] == "GET":
+            request = httpx.Request("GET", url, params=fields)
+        else:
+            request = httpx.Request(scheme["method"], url, json=_nested(fields))
+        request.headers["Accept"] = "application/json"
+        request.extensions["workflowval_login"] = True
+        response = yield request
+        self.token = str(_value_at(response.json(), scheme["token_path"]))
+        _secrets.add(self.token)
+
+    def _apply(self, request: httpx.Request) -> None:
+        scheme = self.scheme
+        request.headers[scheme["token_header"]] = f"{scheme['token_prefix']}{self.token}"
+
+    def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
+        if self.token is None:
+            yield from self._login()
+        self._apply(request)
+        response = yield request
+        if response.status_code == 401 and self.scheme["refresh_on_401"]:
+            yield from self._login()
+            self._apply(request)
+            yield request
+
+
+def _guard(request: httpx.Request) -> None:
+    """Refuse any request to a host that isn't allowed, whatever made it."""
+    if request.url.host not in ALLOWED_HOSTS:
+        raise RuntimeError(f"{request.url.host} isn't an allowed host: {sorted(ALLOWED_HOSTS)}")
+
+
+def _record(response: httpx.Response) -> None:
+    if _recording and not response.request.extensions.get("workflowval_login"):
+        response.read()
+        _exchanges.append(response)
+
+
+def _make_client(*, authenticated: bool) -> httpx.Client:
+    options: dict[str, Any] = {}
+    if authenticated and AUTH:
+        kind = AUTH["type"]
+        if kind == "bearer":
+            options["headers"] = {"Authorization": f"Bearer {_env(AUTH['token_env'])}"}
+        elif kind == "basic":
+            options["auth"] = httpx.BasicAuth(_env(AUTH["username_env"]), _env(AUTH["password_env"]))
+        elif kind == "api_key" and AUTH["location"] == "header":
+            options["headers"] = {AUTH["name"]: _env(AUTH["value_env"])}
+        elif kind == "api_key":
+            options["params"] = {AUTH["name"]: _env(AUTH["value_env"])}
+        elif kind == "login":
+            options["auth"] = _LoginAuth(AUTH)
+    client = httpx.Client(
+        timeout=TIMEOUT_SECONDS,
+        event_hooks={"request": [_guard], "response": [_record]},
+        **options,
+    )
+    client.headers["Accept"] = "application/json"
+    return client
+
+
+class Api:
+    """What a generated test uses to call the API."""
+
+    def __init__(self, client: httpx.Client, anonymous: httpx.Client) -> None:
+        self._client = client
+        self._anonymous = anonymous
+        self.unique = uuid.uuid4().hex[:12]
+        self.unique_int = secrets.randbelow(2**31 - 2) + 1
+
+    def call(
+        self,
+        method: str,
+        template: str,
+        *,
+        path: dict[str, Any] | None = None,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, Any] | None = None,
+        body: Any = None,
+        auth: bool = True,
+    ) -> httpx.Response:
+        """Send one request. `auth=False` sends it without credentials."""
+        global _steps_run
+        if READ_ONLY and method.upper() not in SAFE_METHODS:
+            pytest.skip(f"read-only run: {method} {template} wasn't sent")
+        if _recording:
+            _steps_run += 1
+        url = template
+        for name, value in (path or {}).items():
+            url = url.replace("{" + name + "}", quote(str(value), safe=""))
+        request_headers = {name: str(value) for name, value in (headers or {}).items()}
+        client = self._client if auth else self._anonymous
+        options: dict[str, Any] = {"params": params, "headers": request_headers}
+        if body is not None:
+            options["json"] = body
+        return client.request(
+            method, BASE_URL + url, extensions={"workflowval_template": template}, **options
+        )
+
+    def known_issue(self, reason: str) -> None:
+        """Note that this test has a failure already reviewed as real."""
+        global _known_issue
+        _known_issue = reason
+
+    def expected_failure(self) -> None:
+        """End the test as an expected failure, with the reviewed reason."""
+        pytest.xfail(_known_issue or "known issue")
+
+    @contextmanager
+    def expected_failure_on_assert(self) -> Iterator[None]:
+        """Treat a failed check inside this block as the reviewed failure."""
+        try:
+            yield
+        except AssertionError:
+            self.expected_failure()
+
+    def cleanup(self, method: str, template: str, **options: Any) -> None:
+        """Remove test data. A failure here must not hide the test's own result."""
+        global _recording
+        if READ_ONLY and method.upper() not in SAFE_METHODS:
+            return
+        _recording = False
+        try:
+            self.call(method, template, **options)
+        except httpx.HTTPError:
+            pass
+        finally:
+            _recording = True
+
+    def value(self, response: httpx.Response, path: str) -> Any:
+        """Read a value from a JSON response, like $.a.b[0]."""
+        try:
+            document = response.json()
+        except ValueError as exc:
+            raise AssertionError(f"response isn't JSON: {response.text[:200]}") from exc
+        return _value_at(document, path)
+
+    def fill(self, text: str, saved: dict[str, Any]) -> str:
+        """Replace {placeholders} inside a longer string."""
+        known = {**saved, "unique": self.unique, "unique_int": self.unique_int}
+        return _PLACEHOLDER.sub(lambda match: str(known[match.group(1)]), text)
+
+
+@pytest.fixture
+def api() -> Generator[Api, None, None]:
+    """One API helper per test, with fresh unique values."""
+    global _steps_run, _known_issue
+    _exchanges.clear()
+    _steps_run = 0
+    _known_issue = None
+    with _make_client(authenticated=True) as client, _make_client(authenticated=False) as anon:
+        yield Api(client, anon)
+
+
+def _body(content: bytes) -> Any:
+    text = _mask(content.decode("utf-8", errors="replace"))[:4000]
+    try:
+        return json.loads(text)
+    except ValueError:
+        return text
+
+
+def _headers(headers: httpx.Headers) -> dict[str, str]:
+    return {
+        name: MASK if name.lower() in SECRET_HEADERS else _mask(value)
+        for name, value in headers.items()
+    }
+
+
+def _exchange(response: httpx.Response) -> dict[str, Any]:
+    request = response.request
+    return {
+        "method": request.method,
+        "url": _mask(str(request.url)),
+        "request_headers": _headers(request.headers),
+        "request_body": _body(request.content) if request.content else None,
+        "status": response.status_code,
+        "response_headers": _headers(response.headers),
+        "response_body": _body(response.content) if response.content else None,
+    }
+
+
+def _call(response: httpx.Response) -> dict[str, Any]:
+    """One request and its whole response, for checking against the spec."""
+    raw_body = response.content.decode("utf-8", errors="replace")
+    return {
+        **_exchange(response),
+        "template": response.request.extensions.get("workflowval_template"),
+        "content_type": response.headers.get("content-type"),
+        "raw_body": (raw_body if KEEP_RAW_BODIES else _mask(raw_body))[:_MAX_RAW_BODY],
+    }
+
+
+def pytest_runtest_logreport(report: pytest.TestReport) -> None:
+    """Record each test's outcome and its last 3 HTTP exchanges."""
+    if report.when == "call" or report.outcome == "failed":
+        crash = getattr(report.longrepr, "reprcrash", None)
+        message = crash.message if crash else str(report.longrepr or "")
+        _results[report.nodeid] = {
+            "status": "xfailed" if hasattr(report, "wasxfail") else report.outcome,
+            "duration_s": report.duration,
+            "failing_assertion": _mask(message) if report.failed else None,
+            "stdout": _mask(report.capstdout),
+            "exchanges": [_exchange(response) for response in _exchanges[-3:]],
+            "calls": [_call(response) for response in _exchanges],
+            "steps_run": _steps_run,
+            "known_issue": _mask(_known_issue) if _known_issue else None,
+        }
+
+
+def pytest_collectreport(report: pytest.CollectReport) -> None:
+    """Record test files that can't even be imported."""
+    if report.failed:
+        _results[report.nodeid] = {
+            "status": "error",
+            "duration_s": 0.0,
+            "failing_assertion": _mask(str(report.longrepr))[-2000:],
+            "stdout": "",
+            "exchanges": [],
+        }
+
+
+def pytest_terminal_summary(terminalreporter: Any) -> None:
+    """Say when a test marked as a known issue passed, so the bug may be fixed."""
+    for node_id, result in _results.items():
+        if result["status"] == "passed" and result.get("known_issue"):
+            terminalreporter.write_line(f"{node_id} passed; this may be fixed: {result['known_issue']}")
+
+
+def pytest_sessionfinish() -> None:
+    """Write results for workflowval, when it's the one running these tests."""
+    path = os.environ.get("WORKFLOWVAL_CAPTURE")
+    if path:
+        with open(path, "w", encoding="utf-8") as handle:
+            json.dump(_results, handle, default=str)