pl-e2e-test 0.1.0__py3-none-any.whl

prairielearn_e2e/reporting.py

@@ -0,0 +1,675 @@
+"""Shard report recording, validation, merging, and JUnit output."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+import tempfile
+import time
+import xml.etree.ElementTree as ET
+from collections import Counter, defaultdict
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from .sharding import Shard
+
+REPORT_SCHEMA_VERSION = 1
+REPORT_GLOB = "*.pl-e2e-report.json"
+
+
+class ReportError(ValueError):
+    """A shard report cannot be safely merged."""
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def collection_fingerprint(nodeids: Sequence[str]) -> str:
+    """Return a stable fingerprint for an ordered pytest collection."""
+
+    digest = hashlib.sha256()
+    for nodeid in nodeids:
+        digest.update(nodeid.encode("utf-8"))
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+def serialize_pytest_report(
+    report: pytest.TestReport | pytest.CollectReport,
+) -> dict[str, Any]:
+    longrepr = None if report.longrepr is None else str(report.longrepr)
+    serialized: dict[str, Any] = {
+        "nodeid": report.nodeid,
+        "outcome": report.outcome,
+        "longrepr": longrepr,
+        "sections": [list(section) for section in report.sections],
+    }
+    when = getattr(report, "when", None)
+    if when is not None:
+        serialized["when"] = when
+    duration = getattr(report, "duration", None)
+    if duration is not None:
+        serialized["duration"] = duration
+    wasxfail = getattr(report, "wasxfail", None)
+    if wasxfail is not None:
+        serialized["wasxfail"] = str(wasxfail)
+    return serialized
+
+
+def _atomic_json(path: Path, value: dict[str, Any]) -> None:
+    path = path.resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary_name: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            encoding="utf-8",
+            dir=path.parent,
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as temporary:
+            temporary_name = temporary.name
+            json.dump(value, temporary, indent=2, sort_keys=True)
+            temporary.write("\n")
+        os.replace(temporary_name, path)
+    finally:
+        if temporary_name is not None:
+            try:
+                Path(temporary_name).unlink()
+            except FileNotFoundError:
+                pass
+
+
+@dataclass(slots=True)
+class ShardReportRecorder:
+    """Mutable pytest-session state used to create one shard report."""
+
+    path: Path
+    shard: Shard
+    fuzz_seed: int | None
+    started_at: str = field(default_factory=_now)
+    started_monotonic: float = field(default_factory=time.monotonic)
+    full_nodeids: list[str] = field(default_factory=list)
+    selected_nodeids: list[str] = field(default_factory=list)
+    fingerprint: str = ""
+    reports: list[dict[str, Any]] = field(default_factory=list)
+    collection_errors: list[dict[str, Any]] = field(default_factory=list)
+    warnings: list[dict[str, Any]] = field(default_factory=list)
+    internal_errors: list[str] = field(default_factory=list)
+    interrupted: list[str] = field(default_factory=list)
+    intentionally_empty: bool = False
+
+    def set_collection(
+        self, full_nodeids: Sequence[str], selected_nodeids: Sequence[str]
+    ) -> None:
+        self.full_nodeids = list(full_nodeids)
+        self.selected_nodeids = list(selected_nodeids)
+        self.fingerprint = collection_fingerprint(self.full_nodeids)
+        self.intentionally_empty = bool(self.full_nodeids) and not self.selected_nodeids
+
+    def payload(self, exit_status: int) -> dict[str, Any]:
+        return {
+            "schema_version": REPORT_SCHEMA_VERSION,
+            "producer": {
+                "name": "pl-e2e-test",
+                "version": version("pl-e2e-test"),
+                "pytest_version": pytest.__version__,
+            },
+            "shard": {"index": self.shard.index, "count": self.shard.count},
+            "suite": {
+                "fingerprint": self.fingerprint,
+                "full_nodeids": self.full_nodeids,
+                "selected_nodeids": self.selected_nodeids,
+            },
+            "fuzz_seed": self.fuzz_seed,
+            "started_at": self.started_at,
+            "finished_at": _now(),
+            "duration": time.monotonic() - self.started_monotonic,
+            "exit_status": exit_status,
+            "reports": self.reports,
+            "collection_errors": self.collection_errors,
+            "warnings": self.warnings,
+            "internal_errors": self.internal_errors,
+            "interrupted": self.interrupted,
+        }
+
+    def write(self, exit_status: int) -> None:
+        _atomic_json(self.path, self.payload(exit_status))
+
+
+def write_empty_shard_report(
+    path: Path, shard: Shard, *, fuzz_seed: int | None = None
+) -> None:
+    """Write a successful report for a diff run that selected no questions."""
+
+    recorder = ShardReportRecorder(path=path, shard=shard, fuzz_seed=fuzz_seed)
+    recorder.set_collection((), ())
+    recorder.write(0)
+
+
+@dataclass(frozen=True, slots=True)
+class ItemResult:
+    nodeid: str
+    outcome: str
+    duration: float
+    detail: str
+    sections: tuple[tuple[str, str], ...]
+    reason: str | None = None
+
+
+def _expect_dict(value: Any, context: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ReportError(f"{context} must be a JSON object")
+    return value
+
+
+def _expect_list(value: Any, context: str) -> list[Any]:
+    if not isinstance(value, list):
+        raise ReportError(f"{context} must be a JSON array")
+    return value
+
+
+def _expect_int(value: Any, context: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ReportError(f"{context} must be an integer")
+    return value
+
+
+def _load_report(path: Path) -> dict[str, Any]:
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise ReportError(f"unable to read {path}: {exc}") from exc
+    report = _expect_dict(raw, os.fspath(path))
+    schema_version = report.get("schema_version")
+    if (
+        isinstance(schema_version, bool)
+        or not isinstance(schema_version, int)
+        or schema_version != REPORT_SCHEMA_VERSION
+    ):
+        raise ReportError(f"{path}: unsupported report schema {schema_version!r}")
+    report["_path"] = os.fspath(path)
+    return report
+
+
+def _report_paths(inputs: Sequence[Path]) -> list[Path]:
+    paths: set[Path] = set()
+    for input_path in inputs:
+        if input_path.is_dir():
+            paths.update(path.resolve() for path in input_path.glob(REPORT_GLOB))
+        elif input_path.is_file():
+            paths.add(input_path.resolve())
+        else:
+            raise ReportError(f"report input does not exist: {input_path}")
+    if not paths:
+        raise ReportError("no shard reports found")
+    return sorted(paths, key=lambda path: path.as_posix())
+
+
+def _string_list(value: Any, context: str) -> list[str]:
+    values = _expect_list(value, context)
+    if not all(isinstance(item, str) for item in values):
+        raise ReportError(f"{context} must contain only strings")
+    return values
+
+
+def _validate_reports(reports: Sequence[dict[str, Any]]) -> list[str]:
+    first = reports[0]
+    first_shard = _expect_dict(first.get("shard"), "shard")
+    count = _expect_int(first_shard.get("count"), "shard.count")
+    if count < 1:
+        raise ReportError("shard.count must be at least 1")
+    first_suite = _expect_dict(first.get("suite"), "suite")
+    full_nodeids = _string_list(first_suite.get("full_nodeids"), "suite.full_nodeids")
+    if len(full_nodeids) != len(set(full_nodeids)):
+        raise ReportError("suite.full_nodeids contains duplicates")
+    fingerprint = first_suite.get("fingerprint")
+    if not isinstance(fingerprint, str) or fingerprint != collection_fingerprint(
+        full_nodeids
+    ):
+        raise ReportError("suite fingerprint does not match full_nodeids")
+
+    producer = _expect_dict(first.get("producer"), "producer")
+    if producer.get("name") != "pl-e2e-test" or not all(
+        isinstance(producer.get(key), str) for key in ("version", "pytest_version")
+    ):
+        raise ReportError("producer metadata is invalid")
+    producer_identity = (producer.get("version"), producer.get("pytest_version"))
+    fuzz_seed = first.get("fuzz_seed")
+    if fuzz_seed is not None and (
+        isinstance(fuzz_seed, bool) or not isinstance(fuzz_seed, int)
+    ):
+        raise ReportError("fuzz_seed must be an integer or null")
+    by_index: dict[int, dict[str, Any]] = {}
+    selected: set[str] = set()
+    for report in reports:
+        path = report.get("_path", "report")
+        shard = _expect_dict(report.get("shard"), f"{path}: shard")
+        index = _expect_int(shard.get("index"), f"{path}: shard.index")
+        report_count = _expect_int(shard.get("count"), f"{path}: shard.count")
+        if report_count != count:
+            raise ReportError(f"{path}: shard count does not match other reports")
+        if not 1 <= index <= count:
+            raise ReportError(f"{path}: shard index {index} is out of range")
+        if index in by_index:
+            raise ReportError(f"duplicate report for shard {index}/{count}")
+        by_index[index] = report
+
+        suite = _expect_dict(report.get("suite"), f"{path}: suite")
+        if suite.get("fingerprint") != fingerprint:
+            raise ReportError(f"{path}: collection fingerprint does not match")
+        if (
+            _string_list(suite.get("full_nodeids"), f"{path}: suite.full_nodeids")
+            != full_nodeids
+        ):
+            raise ReportError(f"{path}: full collection does not match")
+        report_producer = _expect_dict(report.get("producer"), f"{path}: producer")
+        if (
+            report_producer.get("version"),
+            report_producer.get("pytest_version"),
+        ) != producer_identity:
+            raise ReportError(f"{path}: producer versions do not match")
+        if report.get("fuzz_seed") != fuzz_seed:
+            raise ReportError(f"{path}: fuzz seed does not match")
+
+        selected_nodeids = _string_list(
+            suite.get("selected_nodeids"), f"{path}: suite.selected_nodeids"
+        )
+        if len(selected_nodeids) != len(set(selected_nodeids)):
+            raise ReportError(f"{path}: selected_nodeids contains duplicates")
+        duplicates = selected.intersection(selected_nodeids)
+        if duplicates:
+            raise ReportError(f"{path}: duplicate selected test {min(duplicates)}")
+        unknown = set(selected_nodeids).difference(full_nodeids)
+        if unknown:
+            raise ReportError(f"{path}: selected test is absent from full collection")
+        selected.update(selected_nodeids)
+
+    missing_indices = sorted(set(range(1, count + 1)).difference(by_index))
+    if missing_indices:
+        raise ReportError(
+            "missing shard reports: "
+            + ", ".join(f"{index}/{count}" for index in missing_indices)
+        )
+    if len(reports) != count:
+        raise ReportError(f"expected {count} shard reports, found {len(reports)}")
+    if selected != set(full_nodeids):
+        missing = set(full_nodeids).difference(selected)
+        raise ReportError(
+            f"reports do not select the full suite; missing {len(missing)} tests"
+        )
+    return full_nodeids
+
+
+def _phases_by_nodeid(
+    reports: Sequence[dict[str, Any]],
+) -> dict[str, list[dict[str, Any]]]:
+    phases: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for report in reports:
+        path = report.get("_path", "report")
+        suite = _expect_dict(report.get("suite"), f"{path}: suite")
+        selected = set(
+            _string_list(
+                suite.get("selected_nodeids"), f"{path}: suite.selected_nodeids"
+            )
+        )
+        for phase in _expect_list(report.get("reports"), f"{path}: reports"):
+            phase_dict = _expect_dict(phase, f"{path}: report entry")
+            nodeid = phase_dict.get("nodeid")
+            if not isinstance(nodeid, str):
+                raise ReportError(f"{path}: report nodeid must be a string")
+            if nodeid not in selected:
+                raise ReportError(f"{path}: result for unselected test {nodeid}")
+            phases[nodeid].append(phase_dict)
+    for nodeid, node_phases in phases.items():
+        whens = [phase.get("when") for phase in node_phases]
+        if len(whens) != len(set(whens)):
+            raise ReportError(f"duplicate test phase for {nodeid}")
+    return phases
+
+
+def _sections(phases: Iterable[dict[str, Any]]) -> tuple[tuple[str, str], ...]:
+    result: list[tuple[str, str]] = []
+    for phase in phases:
+        for section in _expect_list(phase.get("sections", []), "report sections"):
+            if (
+                isinstance(section, list)
+                and len(section) == 2
+                and all(isinstance(value, str) for value in section)
+            ):
+                result.append((section[0], section[1]))
+    return tuple(result)
+
+
+def _item_result(nodeid: str, phases: Sequence[dict[str, Any]]) -> ItemResult:
+    by_when = {
+        phase.get("when"): phase
+        for phase in phases
+        if phase.get("when") in {"setup", "call", "teardown"}
+    }
+    duration = sum(
+        float(phase.get("duration", 0.0))
+        for phase in phases
+        if isinstance(phase.get("duration", 0.0), int | float)
+    )
+    sections = _sections(phases)
+    setup = by_when.get("setup")
+    call = by_when.get("call")
+    teardown = by_when.get("teardown")
+    failed_phases = [
+        phase
+        for phase in (setup, call, teardown)
+        if phase is not None and phase.get("outcome") == "failed"
+    ]
+    if failed_phases:
+        outcome = (
+            "error"
+            if any(phase.get("when") != "call" for phase in failed_phases)
+            else "failed"
+        )
+        return ItemResult(
+            nodeid,
+            outcome,
+            duration,
+            "\n\n".join(
+                str(phase.get("longrepr") or "test failed") for phase in failed_phases
+            ),
+            sections,
+        )
+    skipped_phase = next(
+        (
+            phase
+            for phase in (setup, call)
+            if phase is not None and phase.get("outcome") == "skipped"
+        ),
+        None,
+    )
+    if skipped_phase is not None:
+        wasxfail = skipped_phase.get("wasxfail")
+        outcome = "xfailed" if wasxfail is not None else "skipped"
+        reason = str(wasxfail or skipped_phase.get("longrepr") or "skipped")
+        return ItemResult(nodeid, outcome, duration, "", sections, reason)
+    if (
+        setup is None
+        or setup.get("outcome") != "passed"
+        or call is None
+        or teardown is None
+    ):
+        return ItemResult(
+            nodeid,
+            "error",
+            duration,
+            "test did not complete all setup, call, and teardown phases",
+            sections,
+        )
+    if call.get("wasxfail") is not None:
+        return ItemResult(
+            nodeid,
+            "xpassed",
+            duration,
+            "",
+            sections,
+            str(call["wasxfail"]),
+        )
+    return ItemResult(nodeid, "passed", duration, "", sections)
+
+
+def _session_errors(
+    reports: Sequence[dict[str, Any]], item_results: dict[str, ItemResult]
+) -> list[ItemResult]:
+    errors: list[ItemResult] = []
+    for report in reports:
+        shard = _expect_dict(report.get("shard"), "shard")
+        label = f"shard-{shard.get('index')}-of-{shard.get('count')}"
+        details: list[str] = []
+        for collection_error in _expect_list(
+            report.get("collection_errors", []), "collection_errors"
+        ):
+            item = _expect_dict(collection_error, "collection error")
+            details.append(str(item.get("longrepr") or "collection failed"))
+        details.extend(
+            str(value)
+            for value in _expect_list(
+                report.get("internal_errors", []), "internal_errors"
+            )
+        )
+        details.extend(
+            str(value)
+            for value in _expect_list(report.get("interrupted", []), "interrupted")
+        )
+        exit_status = _expect_int(report.get("exit_status"), "exit_status")
+        suite = _expect_dict(report.get("suite"), "suite")
+        selected_nodeids = _string_list(
+            suite.get("selected_nodeids"), "suite.selected_nodeids"
+        )
+        test_failure_explains_exit = any(
+            item_results[nodeid].outcome in {"failed", "error"}
+            for nodeid in selected_nodeids
+        )
+        if exit_status != 0 and not details and not test_failure_explains_exit:
+            details.append(f"pytest exited with status {exit_status}")
+        if details:
+            errors.append(
+                ItemResult(
+                    nodeid=f"prairielearn_e2e::{label}",
+                    outcome="error",
+                    duration=0.0,
+                    detail="\n\n".join(details),
+                    sections=(),
+                )
+            )
+    return errors
+
+
+def _summary(results: Sequence[ItemResult]) -> str:
+    counts = Counter(result.outcome for result in results)
+    order = ("failed", "error", "passed", "skipped", "xfailed", "xpassed")
+    parts = [f"{counts[name]} {name}" for name in order if counts[name]]
+    return ", ".join(parts) if parts else "no tests ran"
+
+
+def _print_terminal(results: Sequence[ItemResult], warnings: Sequence[str]) -> None:
+    for result in results:
+        if result.outcome not in {"failed", "error"}:
+            continue
+        print(f"{'=' * 20} {result.outcome.upper()} {result.nodeid} {'=' * 20}")
+        print(result.detail)
+        for heading, content in result.sections:
+            print(f"-------------------- {heading} --------------------")
+            print(content)
+    if warnings:
+        print("==================== warnings summary ====================")
+        for warning in warnings:
+            print(warning)
+    noteworthy = [
+        result
+        for result in results
+        if result.outcome in {"failed", "error", "skipped", "xfailed", "xpassed"}
+    ]
+    if noteworthy:
+        print("==================== short test summary info ====================")
+        for result in noteworthy:
+            print(f"{result.outcome.upper():8} {result.nodeid}")
+    duration = sum(result.duration for result in results)
+    print(
+        f"==================== {_summary(results)} in {duration:.2f}s "
+        "===================="
+    )
+
+
+def _failure_message(detail: str, fallback: str) -> str:
+    lines = [line.strip() for line in detail.splitlines() if line.strip()]
+    return lines[-1] if lines else fallback
+
+
+def _xml_safe(value: str) -> str:
+    return "".join(
+        character
+        if (
+            character in "\t\n\r"
+            or 0x20 <= ord(character) <= 0xD7FF
+            or 0xE000 <= ord(character) <= 0xFFFD
+            or 0x10000 <= ord(character) <= 0x10FFFF
+        )
+        else f"#x{ord(character):02X}"
+        for character in value
+    )
+
+
+def _write_junit(path: Path, results: Sequence[ItemResult]) -> None:
+    counts = Counter(result.outcome for result in results)
+    failures = counts["failed"]
+    errors = counts["error"]
+    skipped = counts["skipped"] + counts["xfailed"]
+    duration = sum(result.duration for result in results)
+    testsuites = ET.Element(
+        "testsuites",
+        {
+            "name": "pl-e2e-test",
+            "tests": str(len(results)),
+            "failures": str(failures),
+            "errors": str(errors),
+            "skipped": str(skipped),
+            "time": f"{duration:.6f}",
+        },
+    )
+    testsuite = ET.SubElement(
+        testsuites,
+        "testsuite",
+        {
+            "name": "pl-e2e-test",
+            "tests": str(len(results)),
+            "failures": str(failures),
+            "errors": str(errors),
+            "skipped": str(skipped),
+            "time": f"{duration:.6f}",
+        },
+    )
+    for result in results:
+        testcase = ET.SubElement(
+            testsuite,
+            "testcase",
+            {
+                "classname": "prairielearn_e2e",
+                "name": _xml_safe(result.nodeid),
+                "time": f"{result.duration:.6f}",
+            },
+        )
+        if result.outcome == "failed":
+            failure = ET.SubElement(
+                testcase,
+                "failure",
+                {"message": _xml_safe(_failure_message(result.detail, "test failed"))},
+            )
+            failure.text = _xml_safe(result.detail)
+        elif result.outcome == "error":
+            error = ET.SubElement(
+                testcase,
+                "error",
+                {"message": _xml_safe(_failure_message(result.detail, "test error"))},
+            )
+            error.text = _xml_safe(result.detail)
+        elif result.outcome in {"skipped", "xfailed"}:
+            ET.SubElement(
+                testcase,
+                "skipped",
+                {"message": _xml_safe(result.reason or "skipped")},
+            )
+
+        stdout = "\n".join(
+            content
+            for heading, content in result.sections
+            if "stdout" in heading.lower()
+        )
+        stderr = "\n".join(
+            content
+            for heading, content in result.sections
+            if "stderr" in heading.lower() or "log" in heading.lower()
+        )
+        if stdout:
+            ET.SubElement(testcase, "system-out").text = _xml_safe(stdout)
+        if stderr:
+            ET.SubElement(testcase, "system-err").text = _xml_safe(stderr)
+
+    ET.indent(testsuites)
+    path = path.resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary_name: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="wb",
+            dir=path.parent,
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as temporary:
+            temporary_name = temporary.name
+            ET.ElementTree(testsuites).write(
+                temporary, encoding="utf-8", xml_declaration=True
+            )
+        os.replace(temporary_name, path)
+    finally:
+        if temporary_name is not None:
+            try:
+                Path(temporary_name).unlink()
+            except FileNotFoundError:
+                pass
+
+
+def merge_reports(inputs: Sequence[Path], junitxml: Path) -> int:
+    """Validate and merge shard reports, returning a process exit status."""
+
+    report_paths = _report_paths(inputs)
+    if junitxml.resolve() in report_paths:
+        raise ReportError("JUnit output path must not overwrite an input report")
+    try:
+        junitxml.unlink(missing_ok=True)
+    except OSError as exc:
+        raise ReportError(f"unable to replace {junitxml}: {exc}") from exc
+    reports = [_load_report(path) for path in report_paths]
+    full_nodeids = _validate_reports(reports)
+    phases = _phases_by_nodeid(reports)
+    results = [_item_result(nodeid, phases.get(nodeid, ())) for nodeid in full_nodeids]
+    results.extend(
+        _session_errors(reports, {result.nodeid: result for result in results})
+    )
+    warnings = sorted(
+        {
+            str(warning.get("message", ""))
+            for report in reports
+            for warning in _expect_list(report.get("warnings", []), "warnings")
+            if isinstance(warning, dict) and warning.get("message")
+        }
+    )
+    _print_terminal(results, warnings)
+    _write_junit(junitxml, results)
+    return int(any(result.outcome in {"failed", "error"} for result in results))
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Merge pl-e2e-test shard reports into one terminal and JUnit report."
+    )
+    parser.add_argument("reports", nargs="+", type=Path)
+    parser.add_argument("--junitxml", type=Path, default=Path("pl-e2e-results.xml"))
+    args = parser.parse_args(argv)
+    try:
+        return merge_reports(args.reports, args.junitxml)
+    except (OSError, ReportError) as exc:
+        print(f"pl-e2e-merge-reports: error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())