python-interactive-courses 1.0.0__py3-none-any.whl

pyics/tests/tests.py

@@ -0,0 +1,440 @@
+import copy
+import enum
+import importlib.util as iutil
+import sys
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from contextvars import ContextVar
+from importlib.machinery import ModuleSpec
+from pathlib import Path
+from types import MappingProxyType, ModuleType
+from typing import Any, ClassVar, Literal, Protocol, Self
+
+import pytest
+from _pytest.assertion.util import assertrepr_compare, dummy_highlighter
+from rich.markup import escape
+
+import pyics.util.json_handling as json
+from pyics.notebook import cells, loader, sections
+from pyics.tests.capture import capture_stdout, provide_input
+from pyics.tests.trace import Trace
+from pyics.util import paths
+from pyics.util.paths import StrPath
+from . import check_specs, check_structure, compare
+
+
+class _Inherit(enum.Enum):
+    INHERIT = "inherit"
+
+
+# the section of a test that wasn't given one (see ``PrintTest`` and ``StructureTest``):
+# it inherits its task's section for its file (see ``TASK_SECTIONS``)
+INHERIT = _Inherit.INHERIT
+type Inherit = Literal[_Inherit.INHERIT]
+
+# The section of each exercise notebook that the task whose tests are running is about
+# (see ``Task.sections``), by the notebook's absolute path. Set while a test runs (see
+# ``PyicsItem.runtest``), so that tests that weren't given a section of their own
+# inherit their task's (see ``Test.resolved_section``), without storing it.
+TASK_SECTIONS: ContextVar[Mapping[Path, str]] = ContextVar(
+    "TASK_SECTIONS", default=MappingProxyType({})
+)
+
+
+class LiveSource(Protocol):
+    """
+    Where tests take the code under test from while they run on code that isn't saved
+    yet (see ``LIVE_SOURCE``), e.g. on the objects a notebook's kernel defines.
+    """
+
+    def module(self, path: StrPath) -> ModuleType | None:
+        """The code of the exercise file at ``path`` as a module, or None if this
+        source doesn't hold it, so that the file is loaded as usual."""
+        ...
+
+
+# Set by a plugin while the tests run on code that isn't saved yet (see ``LiveSource``):
+# ``Test.load_solution`` takes the code of the files it holds from it, and
+# ``PrintTest``s and ``StructureTest``s are skipped, as they run on the saved file.
+LIVE_SOURCE: ContextVar[LiveSource | None] = ContextVar("LIVE_SOURCE", default=None)
+
+
+def _skip_live_run() -> None:
+    """Skip the running test if it runs on code that isn't saved yet (see
+    ``LIVE_SOURCE``)."""
+    if LIVE_SOURCE.get() is not None:
+        pytest.skip("runs when the notebook is saved")
+
+
+class MarkupAssertionError(AssertionError):
+    """
+    ``AssertionError`` whose message is ``rich`` markup, which is rendered in the main
+    panel. Messages of other exceptions are shown as they are.
+    """
+
+
+class Test(ABC):
+    """Base Class for Tests"""
+
+    name: str
+    file_path: StrPath
+    # the section of the notebook ``file_path`` the test is about (see
+    # ``resolved_section``): None for the whole notebook, which is all that tests of
+    # other kinds than ``PrintTest`` and ``StructureTest`` are about
+    section: str | None | Inherit = None
+    # whether the test runs the code in ``file_path``, which can then be traced (see
+    # ``run``)
+    supports_trace: ClassVar[bool] = False
+
+    @abstractmethod
+    def __init__(self): ...
+
+    @classmethod
+    def from_dict(cls, test_dict: dict) -> Self:
+        return cls(**test_dict)
+
+    def __call__(self) -> None:
+        self.run()
+
+    def load_solution(
+        self, *, as_main: bool = False, trace: Trace | None = None
+    ) -> ModuleType:
+        """
+        Run the file at ``file_path`` as a new module, and return it. With ``as_main``,
+        it is run as a script, i.e. with ``__name__ == "__main__"``.
+
+        With ``trace``, the file's code is watched by it (see ``Trace.watch``), so that
+        the calls into it that are run by ``trace`` are traced. A script is traced as a
+        whole, while importing a module isn't.
+
+        A notebook's code cells are run instead (see ``loader.run_notebook``), where an
+        error in a cell doesn't stop the run. While a ``LIVE_SOURCE`` that holds the
+        file is set, its module is returned instead, untraced.
+        """
+        if (source := LIVE_SOURCE.get()) is not None and (
+            module := source.module(self.file_path)
+        ) is not None:
+            return module
+        if cells.is_notebook(self.file_path):
+            return loader.load_module(self.file_path, as_main=as_main, trace=trace)
+
+        spec: ModuleSpec | None = iutil.spec_from_file_location(
+            name="current_solution", location=self.file_path
+        )
+        if not isinstance(spec, ModuleSpec):
+            raise ImportError("Could not import solution file. Has the file been moved?")
+        if spec.loader is None:
+            raise AttributeError("spec.loader cannot be None.")
+
+        module = iutil.module_from_spec(spec)
+        if as_main:
+            module.__name__ = "__main__"
+        sys.modules["current_solution"] = module
+        # compiled from source rather than run by the loader, whose cached bytecode is
+        # stale if the file was saved twice within a second without changing its size
+        source = Path(self.file_path).read_bytes()
+        code = compile(source, self.file_path, "exec", dont_inherit=True)
+        if trace is not None:
+            trace.watch(code)
+        if trace is not None and as_main:
+            trace.run(lambda: exec(code, module.__dict__))  # noqa: S102
+        else:
+            exec(code, module.__dict__)  # noqa: S102
+
+        return module
+
+    def resolved_section(self) -> str | None:
+        """
+        The name of the section of the notebook ``file_path`` the test is about (see
+        ``pyics.notebook.sections``), or None for the whole notebook: its own
+        ``section``, or if it wasn't given one, its task's section for its file while
+        the task's tests run (see ``TASK_SECTIONS``).
+        """
+        if self.section is INHERIT:
+            return TASK_SECTIONS.get().get(paths.absolute(self.file_path))
+        return self.section
+
+    def serialize(self) -> str:
+        """turn this test object into a dictionary for export"""
+        return json.dumps(self)
+
+    @abstractmethod
+    def run(self, trace: Trace | None = None) -> None:
+        """
+        Run the test, raising an ``AssertionError`` if it fails. With ``trace``, the code
+        under test is traced by it, if the test ``supports_trace`` and the test's author
+        allows it.
+        """
+
+
+def _visible(output: str) -> str:
+    """``output`` without the whitespace that isn't visible in a console: at the end of
+    lines, and blank lines at the end."""
+    return "\n".join(line.rstrip() for line in output.split("\n")).rstrip("\n")
+
+
+class PrintTest(Test):
+    """
+    Console output of a file must match the desired output.
+
+    The file is run as a script (with ``__name__ == "__main__"``) until it ends or exits,
+    e.g. by calling ``sys.exit()``. Its ``input()`` calls are answered with ``inputs``,
+    in order, which show up in the output like answers typed into a console. Whitespace
+    that isn't visible in a console is ignored. Only the output of the thread running the
+    test is captured, so that other threads (e.g. the CLI's) can keep writing to the
+    console in the meantime.
+
+    A notebook's code cells are run instead, and their output is compared, which
+    includes what Jupyter shows as text: the ``repr`` of objects passed to ``display``,
+    and of each cell's final expression (unless it is ``None``, or the cell ends with
+    ``;``). It only has to contain the desired output, so that what students output
+    besides (e.g. the value of an expression they left at the end of a cell) doesn't
+    fail the test: ``[1, 2]`` is also found in ``[[1, 2]]``. If no output is desired,
+    there must be none. With a ``section`` (see ``Test.resolved_section``), the whole
+    notebook is run, but only the output of the section's cells is compared. A test
+    that isn't given a ``section`` inherits its task's; None compares the output of
+    all cells.
+
+    While a ``LIVE_SOURCE`` is set, it is skipped.
+    """
+
+    expected: str  # printout is necessarily str
+    inputs: list[str]
+    supports_trace = True
+    # whether the test may be traced, e.g. by students who want to see what their code
+    # does. Also the default for tests stored before it was introduced.
+    traceable: bool = True
+    # only set if the test is given a section, so that tests that inherit their task's
+    # section store nothing (and tests stored before sections inherit it, too)
+    section: str | None | Inherit = INHERIT
+
+    def __init__(
+        self,
+        name: str,
+        file_path: StrPath,
+        expected: str,
+        *,
+        inputs: list[str] = [],
+        traceable: bool = True,
+        section: str | None | Inherit = INHERIT,
+        **kwargs,
+    ):
+        if isinstance(inputs, str):
+            raise TypeError("inputs must be a list, with an answer per input() call")
+        self.name = name
+        self.file_path = file_path
+        self.expected = expected
+        self.inputs = [str(answer) for answer in inputs]
+        self.traceable = traceable
+        if section is not INHERIT:
+            self.section = None if section is None else sections.check_name(section)
+
+    def run(self, trace: Trace | None = None) -> None:
+        _skip_live_run()
+        trace = trace if self.traceable else None
+        is_notebook = cells.is_notebook(self.file_path)
+        with capture_stdout() as buffer, provide_input(self.inputs):
+            if is_notebook:
+                if (section := self.resolved_section()) is not None:
+                    # a missing section fails the test before the notebook is run
+                    sections.get_section(cells.read(self.file_path), section)
+                # each cell's output is captured on its own
+                run = loader.run_notebook(self.file_path, as_main=True, trace=trace)
+                buffer.write(run.output(section=section))
+            else:
+                try:
+                    self.load_solution(as_main=True, trace=trace)
+                except SystemExit:
+                    pass  # the program ended
+        res, expected = _visible(buffer.getvalue()), _visible(self.expected)
+        # a notebook's output only has to contain the expected one, unless there is
+        # none, which any output would contain
+        contains = is_notebook and expected != ""
+
+        if not (expected in res if contains else res == expected):
+            expl = list(
+                assertrepr_compare(
+                    "==",
+                    res,
+                    expected,
+                    verbose=1,
+                    highlighter=dummy_highlighter,
+                    assertion_text_diff_style="ndiff",
+                )
+            ) or [f"{res!r} == {expected!r}"]
+            if contains:
+                # the summary of what was compared, above the diff
+                expl[0] = compare.summary(expected, res, op="in")
+
+            raise AssertionError("\n".join(expl))
+
+
+class SimpleTest(Test):
+    """
+    Output of a function must match the desired output for some parameters.
+
+    The output is compared with ``compare.equal``: by ``==``, except for numpy arrays and
+    pandas objects, which are compared as a whole.
+    """
+
+    function_name: str
+    function_args: list[Any]
+    function_kwargs: dict[str, Any]
+    expected: Any
+    supports_trace = True
+    # whether the test may be traced, which shows its arguments, too. Also the default
+    # for tests stored before it was introduced.
+    traceable: bool = True
+
+    def __init__(
+        self,
+        name: str,
+        file_path: StrPath,
+        function_name: str,
+        expected: Any,
+        *,
+        function_args: list[Any] = [],
+        function_kwargs: dict[str, Any] = {},
+        traceable: bool = True,
+        **kwargs,
+    ):
+        self.name = name
+        self.file_path = file_path
+        self.function_name = function_name
+        self.expected = expected
+        self.function_args = function_args
+        self.function_kwargs = function_kwargs
+        self.traceable = traceable
+
+    def run(self, trace: Trace | None = None) -> None:
+        trace = trace if self.traceable else None
+        m = self.load_solution(trace=trace)
+        func = getattr(m, self.function_name)
+        # copied, so a solution that modifies its arguments can't change them for the
+        # next run of this test
+        args = copy.deepcopy(self.function_args)
+        kwargs = copy.deepcopy(self.function_kwargs)
+        if trace is None:
+            res = func(*args, **kwargs)
+        else:
+            res = trace.run(lambda: func(*args, **kwargs))
+
+        if not compare.equal(res, self.expected):
+            raise AssertionError("\n".join(compare.explain(res, self.expected)))
+
+
+class SpecTest(Test):
+    """
+    Symbols in file match some specification.
+
+    ``expected`` is either a spec dict (see ``check_specs``), or a class or function,
+    which is converted into a spec dict so that the test can be stored as json. If
+    ``by_type`` is set, parameters and attributes are matched by their type only, ignoring
+    their names and order.
+    """
+
+    symbol_name: str
+    expected: check_specs.SpecDict
+    by_type: bool
+
+    def __init__(
+        self,
+        name: str,
+        file_path: StrPath,
+        symbol_name: str,
+        expected: check_specs.SpecType,
+        *,
+        by_type: bool = False,
+        **kwargs,
+    ):
+        self.name = name
+        self.file_path = file_path
+        self.symbol_name = symbol_name
+        self.expected = check_specs.to_spec_dict(expected)
+        self.by_type = by_type
+
+    def run(self, trace: Trace | None = None) -> None:
+        m = self.load_solution()
+        symb = getattr(m, self.symbol_name)
+
+        if diff := check_specs.compare_spec(self.expected, symb, by_type=self.by_type):
+            raise MarkupAssertionError("\n".join(diff))
+
+
+class StructureTest(Test):
+    """
+    Source code uses (or avoids) constructs such as loops or recursion.
+
+    ``rules`` is a rule dict or a list of rule dicts (see ``check_structure``), e.g.
+    ``{"concept": "recursion", "scope": "fib"}``. The file is parsed, but not executed.
+
+    Of a notebook, the code cells are checked (see ``loader.python_source``), and lines
+    are named by their cell, e.g. "cell 3, line 2". With a ``section`` (see
+    ``Test.resolved_section``), only the code cells of that section are checked, and
+    the test fails if one of them can't be parsed. A test that isn't given a
+    ``section`` inherits its task's; None checks all code cells, leaving out those that
+    can't be parsed (they are named if the test fails).
+
+    While a ``LIVE_SOURCE`` is set, it is skipped.
+    """
+
+    rules: list[check_structure.Rule]
+    # only set if the test is given a section, so that tests that inherit their task's
+    # section store nothing (and tests stored before sections inherit it, too)
+    section: str | None | Inherit = INHERIT
+
+    def __init__(
+        self,
+        name: str,
+        file_path: StrPath,
+        rules: list[check_structure.Rule] | check_structure.Rule,
+        *,
+        section: str | None | Inherit = INHERIT,
+        **kwargs,
+    ):
+        self.name = name
+        self.file_path = file_path
+        self.rules = check_structure.validate_rules(rules)
+        if section is not INHERIT:
+            self.section = None if section is None else sections.check_name(section)
+
+    def run(self, trace: Trace | None = None) -> None:
+        _skip_live_run()
+        path = Path(self.file_path)
+        if cells.is_notebook(path):
+            section = self.resolved_section()
+            source = loader.python_source(path, section=section)
+            diff = self._check_notebook(source, path.name, whole=section is None)
+        else:
+            diff = check_structure.check_structure(
+                self.rules, path.read_text(encoding="utf-8"), filename=path.name
+            )
+        if diff:
+            raise MarkupAssertionError("\n".join(diff))
+
+    def _check_notebook(
+        self, source: loader.NotebookSource, filename: str, *, whole: bool = True
+    ) -> list[str]:
+        """
+        Messages for the rules that don't hold for the code cells ``source``. Of the
+        ``whole`` notebook, the cells left out since they can't be parsed are named if
+        a rule doesn't hold, as that may be why. Of a section, they fail the test, like
+        a Python file that can't be parsed.
+        """
+        unparsable = [
+            f"Could not parse cell {cell.number}: {escape(err.msg)} "
+            f"(line {err.lineno})."
+            for cell, err in source.unparsable
+        ]
+        if unparsable and not whole:
+            return unparsable
+        diff = check_structure.check_structure(
+            self.rules, source.source, filename=filename, locate=source.locate
+        )
+        return diff + unparsable if diff else []
+
+
+class SizeTest(Test):
+    """file has a maximum line number or size"""
+
+    def __init__(self, test_dict: dict): ...

pyics/tests/trace.py

@@ -0,0 +1,156 @@
+"""
+Call traces of the code under test, recorded with ``snoop``: every line that is run,
+with the values of the variables it changes, and the calls it makes.
+"""
+
+import re
+import sys
+from collections import deque
+from collections.abc import Callable, Generator, Iterator
+from pathlib import Path
+from types import CodeType
+
+import snoop
+from rich.style import Style
+from rich.text import Text
+from snoop.formatting import DefaultFormatter, Event
+from snoop.tracer import FrameInfo
+from snoop.utils import ArgDefaultDict
+
+# the most lines a trace keeps: its first and last half, and how many were left out in
+# between
+MAX_TRACE_LINES = 300
+
+# e.g. ``__builtins__`` and ``len(__builtins__)``
+_DUNDER = re.compile(r"\b__\w+__\b")
+
+
+class _Lines:
+    """
+    Keeps the first and last of the lines written to it, and counts the ones in between,
+    so that e.g. the trace of an endless loop doesn't grow without bounds.
+    """
+
+    def __init__(self, max_lines: int):
+        self._head_size = max_lines // 2
+        self.head: list[str] = []
+        self.tail: deque[str] = deque(maxlen=max_lines - self._head_size)
+        self.omitted = 0
+        self._partial = ""
+
+    def write(self, s: str) -> None:
+        *lines, self._partial = (self._partial + s).split("\n")
+        for line in lines:
+            self._add(line)
+
+    def _add(self, line: str) -> None:
+        if len(self.head) < self._head_size:
+            self.head.append(line)
+            return
+        if len(self.tail) == self.tail.maxlen:
+            self.omitted += 1
+        self.tail.append(line)
+
+    def flush(self) -> None:
+        if self._partial:
+            self._add(self._partial)
+            self._partial = ""
+
+
+class _FrameInfo(FrameInfo):
+    """``FrameInfo`` leaving out dunder variables, e.g. the ``__builtins__`` of a
+    module."""
+
+    def get_local_reprs(self, *args, **kwargs) -> Generator[tuple[str, object]]:
+        for source, value in super().get_local_reprs(*args, **kwargs):
+            if not _DUNDER.search(source):
+                yield source, value
+
+
+class _Formatter(DefaultFormatter):
+    """
+    ``DefaultFormatter`` naming the files of calls without their directories, and
+    showing where a module starts: at its first line, rather than at line 0, which
+    modules start on before running their first line.
+    """
+
+    def format(self, event: Event) -> str:
+        if event.event == "call" and event.line_no == 0:
+            event.line_no = 1
+            formatted = self.format_lines(event, self.format_start(event))
+        else:
+            formatted = super().format(event)
+        if event.event == "call":
+            path = event.code.co_filename
+            formatted = formatted.replace(f'File "{path}"', f'File "{Path(path).name}"')
+        return formatted
+
+
+def _code_objects(code: CodeType) -> Iterator[CodeType]:
+    """``code`` and the code objects defined in it, e.g. of functions, classes, their
+    methods and lambdas."""
+    yield code
+    for const in code.co_consts:
+        if isinstance(const, CodeType):
+            yield from _code_objects(const)
+
+
+class Trace:
+    """
+    Records the trace of the code it watches (see ``watch``) while it is run (see
+    ``run``): of each call into that code, however deeply nested, but not of other
+    code, such as libraries or pyics itself. Only the thread calling ``run`` is traced.
+
+    At most ``max_lines`` lines are kept (see ``text``).
+    """
+
+    def __init__(self, max_lines: int = MAX_TRACE_LINES):
+        self._lines = _Lines(max_lines)
+        self._config = snoop.Config(
+            out=self._lines.write, columns="", color=True, formatter_class=_Formatter
+        )
+        # every frame of the watched code is a target, so depth 1 traces all of them
+        self._tracer = self._config.snoop(depth=1)
+        self._tracer.frame_infos = ArgDefaultDict(_FrameInfo)
+
+    def watch(self, code: CodeType) -> None:
+        """Trace ``code`` and all code defined in it, e.g. a module's functions."""
+        self._tracer.target_codes.update(_code_objects(code))
+
+    def run[T](self, call: Callable[[], T]) -> T:
+        """Return ``call()``, tracing the watched code it runs."""
+        self._config.thread_local.__dict__.setdefault("depth", -1)
+        previous = sys.gettrace()
+        sys.settrace(self._tracer.trace)
+        try:
+            return call()
+        finally:
+            sys.settrace(previous)
+            self._lines.flush()
+
+    @property
+    def ansi(self) -> str:
+        """The trace as snoop highlighted it, with ANSI escape codes, one trace line per
+        line, so that it can be passed on as text, e.g. to another process (see
+        ``ansi_text``). If it is longer than ``max_lines``, the lines in between its
+        first and last ones are left out, and marked as such."""
+        lines = list(self._lines.head)
+        if self._lines.omitted:
+            omitted = f"... {self._lines.omitted} lines omitted ..."
+            lines.append(Style(dim=True).render(omitted))
+        lines += self._lines.tail
+        return "\n".join(lines)
+
+    @property
+    def text(self) -> Text:
+        """The trace, highlighted (see ``ansi``)."""
+        return ansi_text(self.ansi)
+
+
+def ansi_text(ansi: str) -> Text:
+    """
+    The highlighted text of ``ansi``, text with ANSI escape codes such as a trace (see
+    ``Trace.ansi``). Each line is decoded on its own, so that a color left open at the end
+    of a line doesn't run on into the next.
+    """
+    return Text("\n").join(Text.from_ansi(line) for line in ansi.split("\n"))

pyics/util/discovery.py

@@ -0,0 +1,92 @@
+"""
+Find task files and course manifests below a directory.
+"""
+
+import logging
+import os
+from collections.abc import Iterator
+from pathlib import Path, PurePath
+from typing import Any
+
+from pyics.util import json_handling as json
+from pyics.util import paths
+
+TASK_SUFFIX = ".pyicstask.json"
+COURSE_SUFFIX = ".pyicscourse.json"
+# never searched for task files or course manifests: large, and never contain any
+SKIP_DIRS = {
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "__pycache__",
+    ".pytest_cache",
+    ".ruff_cache",
+}
+
+# a plain logger rather than ``get_logger``, which would set up pyics's log file when
+# courses are looked for outside the TUI, e.g. by a plugin in a notebook's kernel
+_logger = logging.getLogger(__name__)
+
+
+def find_files(root: Path, max_depth: int, suffix: str) -> Iterator[Path]:
+    """The files whose names end in ``suffix`` at most ``max_depth`` directories below
+    ``root``, skipping ``SKIP_DIRS``, relative to ``root``."""
+    for dirpath, dirnames, filenames in os.walk(root):
+        rel = Path(dirpath).relative_to(root)
+        if len(rel.parts) >= max_depth:
+            dirnames.clear()
+        else:
+            dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
+        for filename in sorted(filenames):
+            if filename.endswith(suffix):
+                yield rel / filename
+
+
+def find_task_files(root: Path, max_depth: int) -> dict[Path, str]:
+    """
+    Find task files at most ``max_depth`` directories below ``root``, skipping
+    ``SKIP_DIRS``. Files that can't be read or have no task name are skipped.
+
+    Returns
+    -------
+    dict[Path, str]
+        The task files' paths, relative to ``root``, mapped to their task names.
+    """
+    found: dict[Path, str] = {}
+    for rel in find_files(root, max_depth, TASK_SUFFIX):
+        try:
+            name = json.load_plain(root / rel)["name"]
+        except Exception:  # noqa: BLE001
+            _logger.debug(f"Skipping {root / rel}", exc_info=True)
+            continue
+        if isinstance(name, str):
+            found[rel] = name
+    return found
+
+
+def find_courses(root: Path, max_depth: int) -> dict[Path, dict[str, Any]]:
+    """
+    Find course manifests at most ``max_depth`` directories below ``root``, skipping
+    ``SKIP_DIRS``. Files that can't be read or have no tasks are skipped.
+
+    Returns
+    -------
+    dict[Path, dict[str, Any]]
+        The manifests' absolute paths, mapped to their contents.
+    """
+    found: dict[Path, dict[str, Any]] = {}
+    for rel in find_files(root, max_depth, COURSE_SUFFIX):
+        path = paths.absolute(root / rel)
+        try:
+            # read as plain data, so finding courses never imports a module
+            manifest = json.load_plain(path)
+            tasks = manifest["tasks"]
+        except Exception:  # noqa: BLE001
+            _logger.debug(f"Skipping {path}", exc_info=True)
+            continue
+        if isinstance(tasks, dict) and all(
+            isinstance(v, (str, PurePath)) for v in tasks.values()
+        ):
+            found[path] = manifest
+    return found