python-interactive-courses 1.0.0__py3-none-any.whl

pyics/tests/compare.py

@@ -0,0 +1,184 @@
+"""
+Comparing the result of a function with the expected one, for ``SimpleTest``.
+
+Plain values are compared with ``==``. numpy arrays and pandas objects are compared as a
+whole instead, since their ``==`` compares element by element. numpy and pandas aren't
+dependencies of pyics: their objects are only looked for when they are imported, as
+they are when a result or an expected value is one of them.
+"""
+
+import sys
+from typing import Any
+
+from _pytest._io.saferepr import saferepr
+from _pytest.assertion.util import assertrepr_compare, dummy_highlighter
+
+# length of each side of a summary, as in pytest's: (80 - 15 - len("==") - 2) // 2
+_SUMMARY_SIZE = 30
+# positions listed when arrays differ
+_MAX_POSITIONS = 10
+
+
+class IncomparableError(AssertionError):
+    """A result can't be compared with the expected value, e.g. since its ``==`` gives
+    neither ``True`` nor ``False``."""
+
+
+def _is_array(value: Any) -> bool:
+    np = sys.modules.get("numpy")
+    return np is not None and isinstance(value, np.ndarray)
+
+
+def _pandas_types() -> tuple[type, ...]:
+    pd = sys.modules.get("pandas")
+    return () if pd is None else (pd.DataFrame, pd.Series, pd.Index)
+
+
+def summary(left: Any, right: Any, *, op: str = "==") -> str:
+    """The comparison of ``left`` and ``right`` by ``op`` in a line, with both
+    shortened, e.g. ``'3' in '9'``."""
+    left = saferepr(left, maxsize=_SUMMARY_SIZE)
+    return f"{left} {op} {saferepr(right, maxsize=_SUMMARY_SIZE)}"
+
+
+def _type_name(value: Any) -> str:
+    cls = type(value)
+    if cls.__module__ == "builtins":
+        return cls.__qualname__
+    return f"{cls.__module__}.{cls.__qualname__}"
+
+
+def equal(res: Any, expected: Any) -> bool:
+    """
+    Whether ``res`` matches ``expected``. numpy arrays match if their shapes, the kinds of
+    their values (e.g. integers, see ``numpy.dtype.kind``) and their values do, with NaNs
+    matching NaNs. pandas objects match by their ``equals``. Lists, tuples and dicts are
+    compared item by item, so that they can hold arrays, too. Anything else matches if
+    ``res == expected``.
+
+    Raises ``IncomparableError`` if ``==`` gives neither ``True`` nor ``False``.
+    """
+    if _is_array(res) or _is_array(expected):
+        return not _array_differences(res, expected)
+    pandas_types = _pandas_types()
+    if isinstance(res, pandas_types) or isinstance(expected, pandas_types):
+        return isinstance(expected, pandas_types) and bool(expected.equals(res))
+    if type(res) is type(expected) and type(res) in (list, tuple):
+        return len(res) == len(expected) and all(
+            r is e or equal(r, e) for r, e in zip(res, expected)
+        )
+    if type(res) is type(expected) is dict:
+        return res.keys() == expected.keys() and all(
+            res[k] is expected[k] or equal(res[k], expected[k]) for k in res
+        )
+
+    eq = res == expected
+    try:
+        return bool(eq)
+    except Exception as error:  # e.g. element by element, by an unknown array type
+        raise IncomparableError(
+            f"{summary(res, expected)}\n\nCan't tell whether they are equal: == gave "
+            f"{saferepr(eq, maxsize=_SUMMARY_SIZE)}, not True or False"
+        ) from error
+
+
+def _array_differences(res: Any, expected: Any) -> list[str]:
+    """How ``res`` differs from ``expected``, one of which is a numpy array, as by
+    ``equal``. Empty if they match."""
+    np = sys.modules["numpy"]
+    if not (isinstance(res, np.ndarray) and isinstance(expected, np.ndarray)):
+        return [f"Types differ: {_type_name(res)} != {_type_name(expected)}"]
+    if res.shape != expected.shape:
+        return [f"Shapes differ: {res.shape} != {expected.shape}"]
+    if res.dtype.kind != expected.dtype.kind:
+        return [f"Types of values differ: {res.dtype} != {expected.dtype}"]
+
+    if res.dtype.kind == "O":  # any objects, e.g. arrays themselves
+        same = np.frompyfunc(equal, 2, 1)(res, expected)
+    else:
+        same = res == expected
+        if res.dtype.kind in "fc":
+            same |= np.isnan(res) & np.isnan(expected)
+    # all positions differ if the values weren't compared one by one
+    same = np.broadcast_to(np.asarray(same, dtype=bool), res.shape)
+    if same.all():
+        return []
+    if res.ndim == 0:
+        return [f"Values differ: {res.item()!r} != {expected.item()!r}"]
+
+    count = int(np.count_nonzero(~same))
+    lines = [f"Values differ at {count} of {same.size} positions:"]
+    for position in np.argwhere(~same)[:_MAX_POSITIONS]:
+        p = tuple(int(i) for i in position)
+        index = p[0] if len(p) == 1 else p
+        lines.append(f"At index {index} diff: {res.item(p)!r} != {expected.item(p)!r}")
+    if count > _MAX_POSITIONS:
+        lines.append(f"... and {count - _MAX_POSITIONS} more")
+    return lines
+
+
+def _pandas_differences(res: Any, expected: Any) -> list[str]:
+    """How ``res`` differs from ``expected``, one of which is a pandas object, as told by
+    ``pandas.testing``."""
+    pd = sys.modules["pandas"]
+    checks = {
+        pd.DataFrame: pd.testing.assert_frame_equal,
+        pd.Series: pd.testing.assert_series_equal,
+        pd.Index: pd.testing.assert_index_equal,
+    }
+    for kind, check in checks.items():
+        if isinstance(res, kind) and isinstance(expected, kind):
+            break
+    else:
+        return [f"Types differ: {_type_name(res)} != {_type_name(expected)}"]
+    try:
+        check(res, expected, check_exact=True)
+    except AssertionError as error:
+        return str(error).strip().splitlines()
+    return []
+
+
+class _Whole:
+    """An array or pandas object in a list, tuple or dict, which pytest then compares
+    as a whole, by ``equal``, rather than element by element."""
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __init__(self, value: Any):
+        self.value = value
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, _Whole) and equal(self.value, other.value)
+
+    def __repr__(self) -> str:
+        return repr(self.value)
+
+
+def _wrap(value: Any) -> Any:
+    if _is_array(value) or isinstance(value, _pandas_types()):
+        return _Whole(value)
+    if type(value) in (list, tuple):
+        return type(value)(_wrap(v) for v in value)
+    if type(value) is dict:
+        return {k: _wrap(v) for k, v in value.items()}
+    return value
+
+
+def explain(res: Any, expected: Any) -> list[str]:
+    """Lines explaining how ``res`` differs from ``expected``: a summary, followed by
+    the details, e.g. the positions at which arrays differ."""
+    if _is_array(res) or _is_array(expected):
+        details = _array_differences(res, expected)
+    elif isinstance(res, _pandas_types()) or isinstance(expected, _pandas_types()):
+        details = _pandas_differences(res, expected)
+    else:
+        lines = assertrepr_compare(
+            "==",
+            _wrap(res),
+            _wrap(expected),
+            verbose=1,
+            highlighter=dummy_highlighter,
+            assertion_text_diff_style="ndiff",
+        )
+        return list(lines) or [f"{res!r} == {expected!r}"]
+    return [summary(res, expected), *(["", *details] if details else [])]

pyics/tests/pytest_integration.py

@@ -0,0 +1,228 @@
+import dataclasses
+from collections.abc import Mapping
+from os import PathLike
+from pathlib import Path
+from types import MappingProxyType
+from typing import Literal
+
+import pytest
+from pytest import TestReport
+from _pytest._code.code import TerminalRepr
+from _pytest._io import TerminalWriter
+from _pytest.nodes import Node
+from rich.errors import MarkupError
+from rich.style import Style
+from rich.text import Text
+
+from pyics.notebook import loader
+from pyics.tests import Test
+from pyics.tests.results import TestResult
+from pyics.tests.tests import TASK_SECTIONS, MarkupAssertionError
+from pyics.tests.trace import Trace, ansi_text
+from pyics.util import paths
+from pyics.util.paths import StrPath
+
+
+@dataclasses.dataclass(eq=False)
+class RichFailureRepr(TerminalRepr):
+    """
+    Failure representation of a ``PyicsItem``, keeping the highlighting of its message
+    (and its ``trace``, if the test was traced) for the main panel. Elsewhere, e.g. in
+    ``TestReport.longreprtext``, it is plain text.
+    """
+
+    text: Text
+    # the trace as text with ANSI escape codes (see ``Trace.ansi``), which a
+    # ``TestResult`` keeps as it is
+    trace_ansi: str | None = None
+
+    @property
+    def trace(self) -> Text | None:
+        """The trace of the failed test, or why there is none, if it was traced."""
+        return None if self.trace_ansi is None else ansi_text(self.trace_ansi)
+
+    def toterminal(self, tw: TerminalWriter) -> None:
+        tw.line(self.text.plain)
+        if (trace := self.trace) is not None:
+            tw.line(trace.plain)
+
+
+class PyicsItem(pytest.Item):
+    """
+    pytest.Item intended to wrap the tests below for access with pytest
+
+    """
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        parent: Node,
+        test_obj: Test,
+        tracing: bool = False,
+        sections: Mapping[StrPath, str] | None = None,
+    ):
+        self.test_obj = test_obj
+        # whether the test is traced, if it can be (see ``Test.run``), to show the trace
+        # if it fails
+        self.tracing = tracing
+        # the sections of its task, which it inherits while it runs (see
+        # ``TASK_SECTIONS``)
+        self.sections: Mapping[Path, str] = MappingProxyType(
+            {paths.absolute(path): name for path, name in (sections or {}).items()}
+        )
+        self.recorded_trace: Trace | None = None
+        # the errors in the cells of a notebook the test ran, named if it fails
+        self.cell_errors: list[loader.CellError] = []
+        super().__init__(name=self.test_obj.name, parent=parent)
+
+    def runtest(self) -> None:
+        loader.clear_last_errors()
+        task_sections = TASK_SECTIONS.set(self.sections)
+        try:
+            if not (self.tracing and self.test_obj.supports_trace):
+                self.test_obj.run()
+                return
+            # tests whose authors don't allow it ignore the trace
+            self.recorded_trace = Trace()
+            self.test_obj.run(trace=self.recorded_trace)
+        finally:
+            self.cell_errors = loader.last_errors()
+            TASK_SECTIONS.reset(task_sections)
+
+    def repr_failure(
+        self,
+        excinfo: pytest.ExceptionInfo[BaseException],
+        style: None
+        | Literal["long"]
+        | Literal["short"]
+        | Literal["line"]
+        | Literal["no"]
+        | Literal["native"]
+        | Literal["value"]
+        | Literal["auto"] = None,
+    ) -> str | TerminalRepr:
+        message = Text(str(excinfo.value))
+        if isinstance(excinfo.value, MarkupAssertionError):
+            try:
+                message = Text.from_markup(str(excinfo.value), emoji=False)
+            except MarkupError:
+                pass  # shown as it is, rather than failing the whole run
+        for error in self.cell_errors:  # which may be why the test failed
+            message = Text.assemble(message, "\n", error.describe())
+        return RichFailureRepr(
+            Text.assemble(f"{self.name} failed: ", message), self._trace_ansi()
+        )
+
+    def _trace_ansi(self) -> str | None:
+        """The trace of the failed test, or why there is none (dimmed), if it is
+        traced, as text with ANSI escape codes (see ``Trace.ansi``)."""
+        if not self.tracing:
+            return None
+        dim = Style(dim=True).render
+        if not self.test_obj.supports_trace:
+            return dim(f"{type(self.test_obj).__name__}s can't be traced.")
+        if not getattr(self.test_obj, "traceable", True):
+            return dim("Tracing is turned off for this test.")
+        assert self.recorded_trace is not None
+        if not (trace := self.recorded_trace.ansi):
+            return dim(
+                "Nothing was traced: the test failed before running your code."
+            )
+        return trace
+
+    def reportinfo(self) -> tuple[PathLike | str, int | None, str]:
+        return self.path, 0, f"pyics test: {self.name}"
+
+
+class LiveTestCollectionPlugin:
+    """
+    Injects pre-built Test objects as PyicsItems, bypassing pytest's file-based collection.
+
+    This plugin is used for the main pyics script.
+    """
+
+    def __init__(
+        self,
+        test_objs: list[Test],
+        *,
+        tracing: bool = False,
+        sections: Mapping[StrPath, str] | None = None,
+    ):
+        """
+        Parameters
+        ----------
+        test_objs: list[Test]
+            The tests to run.
+        tracing: bool (optional, keyword only)
+            Trace the tests that can be traced, to show their traces if they fail (see
+            ``PyicsItem``). Default is False.
+        sections: Mapping[StrPath, str] | None (optional, keyword only)
+            The section of each exercise notebook that the tests' task is about (see
+            ``Task.sections``), which tests that weren't given a section of their own
+            inherit (see ``Test.resolved_section``). Default is None, i.e. none.
+        """
+        self.test_objs = test_objs
+        self.tracing = tracing
+        self.sections = sections
+        self.reports = []
+        # the reports as results that can cross a process boundary, with the position
+        # of their test in ``test_objs``
+        self.results: list[TestResult] = []
+        self._indices: dict[int, int] = {}
+        self._index: int | None = None
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_collection(self, session: pytest.Session):
+        session.items = [
+            PyicsItem.from_parent(
+                session,
+                name="",
+                test_obj=obj,
+                tracing=self.tracing,
+                sections=self.sections,
+            )
+            for obj in self.test_objs
+        ]
+        session.testscollected = len(session.items)
+        # by identity, since tests can share their name, and with it their nodeid
+        self._indices = {id(item): i for i, item in enumerate(session.items)}
+        return (
+            True  # @Claude: "short-circuits the default collection, no files are touched"
+        )
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item | None):
+        # the reports logged until the next test starts are this test's
+        self._index = self._indices.get(id(item))
+
+    def pytest_runtest_logreport(self, report):
+        self.reports.append(report)
+        self.results.append(TestResult.from_report(report, self._index))
+
+
+class NotebookCollectionPlugin:
+    """
+    Injects pre-built Test objects as PyicsItems, bypassing pytest's file-based collection.
+
+    This plugin is meant to be used in Jupyter Notebooks, e.g. the create_task notebook.
+    """
+
+    def __init__(self, test_objs, *, tracing: bool = False):
+        self.test_objs = test_objs
+        self.tracing = tracing
+        self.reports = []
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_collection(self, session):
+        session.items = [
+            PyicsItem.from_parent(
+                session, name=f"test_{i}", test_obj=obj, tracing=self.tracing
+            )
+            for i, obj in enumerate(self.test_objs)
+        ]
+        session.testscollected = len(session.items)
+        return True  # short-circuits the default collection, no files are touched
+
+    def pytest_runtest_logreport(self, report):
+        self.reports.append(report)

pyics/tests/results.py

@@ -0,0 +1,145 @@
+"""
+Test results as plain data, which can cross a process boundary: written as JSON by the
+process that ran the tests (e.g. a notebook's kernel, see ``pyics.plugins``), and shown
+by another (the TUI) as if it had run them itself.
+"""
+
+import dataclasses
+import math
+from collections.abc import Mapping
+from typing import Any, Self
+
+import pytest
+from rich.errors import MarkupError
+from rich.markup import escape
+from rich.text import Text
+
+from pyics.tests.trace import ansi_text
+
+def _field(data: Mapping[str, Any], name: str, *types: type) -> Any:
+    """``data[name]``, which must be of one of ``types`` (a missing key counts as None).
+    JSON's ``true`` and ``false`` don't count as numbers."""
+    value = data.get(name)
+    if not isinstance(value, types) or (isinstance(value, bool) and bool not in types):
+        raise ValueError(f"{name!r} must be {' or '.join(t.__name__ for t in types)}")
+    return value
+
+
+@dataclasses.dataclass(frozen=True)
+class TestResult:
+    """
+    The result of one phase (setup, call or teardown) of one test, as the main panel
+    shows it (see ``Task.update_status``), in a form that can be stored as JSON (see
+    ``to_json`` and ``from_json``).
+
+    Attributes
+    ----------
+    nodeid: str
+        pytest's id of the test, e.g. "::adds".
+    when: str
+        The phase: "setup", "call" or "teardown".
+    outcome: str
+        "passed", "failed" or "skipped".
+    duration: float
+        How long the phase took, in seconds.
+    message: str | None
+        Why the test failed or was skipped, as rich console markup, e.g. "adds failed:
+        3 == 4" (see ``message_text``). None if it passed.
+    trace: str | None
+        If the test failed while it was traced: its trace, or a line saying why there
+        is none, as text with ANSI escape codes (see ``Trace.ansi`` and
+        ``trace_text``). None if it wasn't traced.
+    index: int | None
+        The position of the test in the tests that were run, e.g. to match the results
+        of different runs of a task's tests. None if unknown.
+    """
+
+    __test__ = False  # not a test class, despite its name
+
+    nodeid: str
+    when: str
+    outcome: str
+    duration: float
+    message: str | None = None
+    trace: str | None = None
+    index: int | None = None
+
+    @property
+    def message_text(self) -> Text:
+        """``message`` as highlighted text, empty if there is none. Malformed markup is
+        shown as it is."""
+        if self.message is None:
+            return Text()
+        try:
+            return Text.from_markup(self.message, emoji=False)
+        except MarkupError:
+            return Text(self.message)
+
+    @property
+    def trace_text(self) -> Text | None:
+        """``trace`` as highlighted text, or None if there is none."""
+        return None if self.trace is None else ansi_text(self.trace)
+
+    def to_json(self) -> dict[str, Any]:
+        """The result as a dict of JSON types, the inverse of ``from_json``."""
+        return dataclasses.asdict(self)
+
+    @classmethod
+    def from_json(cls, data: Any) -> Self:
+        """
+        The result stored as ``data`` by ``to_json``. Keys it doesn't know are ignored.
+
+        Raises
+        ------
+        ValueError
+            If ``data`` isn't a stored result.
+        """
+        if not isinstance(data, Mapping):
+            raise ValueError("a test result must be an object")
+        duration = float(_field(data, "duration", int, float))
+        if not math.isfinite(duration):
+            raise ValueError("'duration' must be finite")
+        return cls(
+            nodeid=_field(data, "nodeid", str),
+            when=_field(data, "when", str),
+            outcome=_field(data, "outcome", str),
+            duration=duration,
+            message=_field(data, "message", str, type(None)),
+            trace=_field(data, "trace", str, type(None)),
+            index=_field(data, "index", int, type(None)),
+        )
+
+    @classmethod
+    def from_report(cls, report: pytest.TestReport, index: int | None = None) -> Self:
+        """
+        The result of the phase of a test ``report`` reports on, keeping the
+        highlighting of the message and the trace of a ``PyicsItem``.
+
+        Parameters
+        ----------
+        report: pytest.TestReport
+            The report.
+        index: int | None
+            The position of the test in the tests that were run (see ``index``).
+        """
+        # imported here, because pytest_integration builds results with this class
+        from pyics.tests.pytest_integration import RichFailureRepr
+
+        message = trace = None
+        if isinstance(report.longrepr, RichFailureRepr):
+            message = report.longrepr.text.markup
+            trace = report.longrepr.trace_ansi
+        elif report.skipped and isinstance(report.longrepr, tuple):
+            # (file, line, reason) of a skipped test
+            message = escape(str(report.longrepr[-1]))
+        elif not report.passed:
+            message = escape(report.longreprtext)
+        return cls(
+            nodeid=report.nodeid,
+            when=report.when,
+            outcome=report.outcome,
+            duration=float(report.duration),
+            message=message,
+            trace=trace,
+            index=index,
+        )