im-pytest 0.1.18__py3-none-any.whl

im_pytest/__init__.py

@@ -0,0 +1,24 @@
+"""im-pytest — run the course's project tests three ways.
+
+* ``check("translationproject")`` or ``%%test translationproject`` — friendly,
+  hidden test runner for the early weeks (mode 1).
+* ``pytest test_translationproject.py`` — the raw pytest CLI, once students learn
+  to read its output (mode 2).
+* students write their own ``assert`` tests to validate AI-produced code (mode 3).
+
+Importing the package registers the ``%%test`` cell magic when run in IPython.
+"""
+from __future__ import annotations
+
+from .report import Report, Outcome
+from .runner import run, run_injected
+from .widget import check, register_test_magic, TestResultWidget
+
+__all__ = ["check", "run", "run_injected", "Report", "Outcome",
+           "TestResultWidget", "register_test_magic"]
+
+# Auto-register the cell magic in notebooks (mirrors steps-widget / script-widget).
+try:  # pragma: no cover
+    register_test_magic()
+except Exception:  # pragma: no cover
+    pass

im_pytest/cli.py

@@ -0,0 +1,43 @@
+"""The ``pytest-check`` command: the terminal way to run a project's tests.
+
+    pytest-check translationproject.py     # point at your solution file
+    pytest-check translationproject        # or just the project name
+
+Finds ``test_<project>.py`` (in the working folder or IM_PROJECT_TESTS), runs it
+against your solution, and prints friendly output.
+"""
+from __future__ import annotations
+
+import os
+import sys
+
+from .resources import resolve_test
+from .runner import run
+
+
+def main(argv=None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not argv or argv[0] in ("-h", "--help"):
+        print(__doc__)
+        return 0
+
+    target = argv[0]
+    all_tests = "--all" in argv[1:]
+
+    project = os.path.basename(target)
+    if project.endswith(".py"):
+        project = project[:-3]
+
+    try:
+        test_path = resolve_test(project)
+    except FileNotFoundError as exc:
+        print(exc)
+        return 2
+
+    report = run(test_path, project=project, failfast=not all_tests)
+    print(report.to_text())
+    return 0 if report.ok else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

im_pytest/plugin.py

@@ -0,0 +1,152 @@
+"""The pytest plugin that makes the course test files work.
+
+Registered as a ``pytest11`` entry point, so it is active whenever pytest runs
+against an installed ``im_pytest`` — which is what makes *raw* ``pytest
+test_<project>.py`` (course mode 2) work with no boilerplate in the test file.
+
+It provides three things the per-project test files rely on:
+
+* a ``sol`` fixture   — the student's solution, imported fresh from the working
+  directory (so edits are picked up on re-run) with the student's own ``print``
+  output suppressed;
+* a ``requires`` marker — ``@pytest.mark.requires("translate_codon", ...)`` skips
+  a test when the student has not defined (or has misspelled) a needed name, and
+  collects the missing names;
+* a "functions not defined" summary at the end of a raw pytest run.
+"""
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import os
+import sys
+import types
+
+import pytest
+
+# %%test injects a module here so the fixture can return an in-notebook cell
+# instead of a file on disk. Keyed by module name.
+_INJECTED: dict[str, types.ModuleType] = {}
+
+
+# --------------------------------------------------------------------------- #
+# importing the student's solution
+# --------------------------------------------------------------------------- #
+
+def student_module_name(test_module_name: str) -> str:
+    """`test_translationproject` -> `translationproject`."""
+    base = test_module_name.rsplit(".", 1)[-1]
+    return base[5:] if base.startswith("test_") else base
+
+
+def import_student(name: str, cwd: str | None = None) -> types.ModuleType:
+    """Import (or re-import) the student's `<name>.py` from `cwd`.
+
+    Loaded by explicit file path (so it can't accidentally resolve to a stale
+    copy left on ``sys.path`` by a previous run), with ``cwd`` also put at the
+    front of ``sys.path`` so a solution that imports a *sibling* project file
+    still works. Raises on any error in the student's code so callers can report
+    it. The student's ``print`` output is *not* suppressed here — the caller
+    (the mode-1 runner, or pytest's own capture in mode 2) captures it so it can
+    be shown in the widget's terminal-output area.
+    """
+    if name in _INJECTED:
+        return _INJECTED[name]
+
+    cwd = os.path.abspath(cwd or os.getcwd())
+    if sys.path and sys.path[0] != cwd:
+        # keep cwd first so sibling-file imports resolve to this folder
+        while cwd in sys.path:
+            sys.path.remove(cwd)
+        sys.path.insert(0, cwd)
+
+    sys.modules.pop(name, None)           # drop any cached copy so edits are seen
+    importlib.invalidate_caches()
+
+    path = os.path.join(cwd, name + ".py")
+    if os.path.exists(path):
+        spec = importlib.util.spec_from_file_location(name, path)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module        # so recursive/self references resolve
+        try:
+            spec.loader.exec_module(module)
+        except Exception:
+            sys.modules.pop(name, None)
+            raise
+        return module
+    # Fall back to a normal import (e.g. an injected/installed module).
+    return importlib.import_module(name)
+
+
+def _get_student(config, name):
+    """Import once per session and cache (module or the exception raised)."""
+    cache = getattr(config, "_im_student_cache", None)
+    if cache is None:
+        cache = config._im_student_cache = {}
+    if name not in cache:
+        try:
+            cache[name] = import_student(name)
+        except Exception as exc:                       # noqa: BLE001
+            cache[name] = exc
+    return cache[name]
+
+
+# --------------------------------------------------------------------------- #
+# pytest hooks
+# --------------------------------------------------------------------------- #
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "requires(*names): skip this test when the student has not defined the named "
+        "functions/variables, and report them as not-yet-defined.",
+    )
+    config._im_undefined = set()
+
+
+@pytest.fixture
+def sol(request):
+    """The student's solution module (fresh, prints suppressed)."""
+    name = student_module_name(request.module.__name__)
+    obj = _get_student(request.config, name)
+    if isinstance(obj, Exception):
+        pytest.fail(
+            "Your code could not be run:\n\n" + "".join(
+                __import__("traceback").format_exception_only(type(obj), obj)
+            ),
+            pytrace=False,
+        )
+    return obj
+
+
+def pytest_collection_modifyitems(config, items):
+    for item in items:
+        marker = item.get_closest_marker("requires")
+        if marker is None:
+            continue
+        name = student_module_name(item.module.__name__)
+        obj = _get_student(config, name)
+        if isinstance(obj, Exception):
+            # the sol fixture will surface the import error; nothing to skip on
+            continue
+        missing = [n for n in marker.args if not hasattr(obj, n)]
+        if missing:
+            config._im_undefined.update(missing)
+            item.add_marker(
+                pytest.mark.skip(reason="not defined: " + ", ".join(missing))
+            )
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    undefined = sorted(getattr(config, "_im_undefined", ()) or [])
+    if undefined:
+        tr = terminalreporter
+        tr.write_line("")
+        tr.write_line("*" * 57)
+        tr.write_line("ATTENTION! These functions are not defined (yet):")
+        tr.write_line("")
+        for n in undefined:
+            tr.write_line("\t" + n)
+        tr.write_line("")
+        tr.write_line("They are either misspelled or not written yet.")
+        tr.write_line("*" * 57)

im_pytest/report.py

@@ -0,0 +1,107 @@
+"""Data model for a test run plus a plain-text rendering (used by the CLI).
+
+The notebook rendering is a real anywidget — see ``widget.py``. Both consume
+*this* object, so the two never drift.
+
+A run has two distinct kinds of result:
+
+* **check results** — one :class:`Outcome` per tested function: it passed, an
+  assertion about its return value failed (``FAIL``), or the student's own code
+  raised while it was being tested (``ERROR``);
+* **terminal output** — anything the student's code printed, plus the traceback
+  of a non-assertion error in their code (or an import/syntax error that stopped
+  the file running at all). This mirrors what the ``%%exercise`` widget shows.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+PASS = "pass"
+FAIL = "fail"     # an assertion about the return value failed
+ERROR = "error"   # the student's code raised while being tested
+
+
+@dataclass
+class Outcome:
+    name: str            # the function being tested, e.g. "translate_codon"
+    status: str          # PASS / FAIL / ERROR
+    message: str = ""    # the failing assertion, or a short "raised X" note
+
+
+@dataclass
+class Report:
+    project: str = ""
+    outcomes: List[Outcome] = field(default_factory=list)
+    undefined: List[str] = field(default_factory=list)  # required names not defined yet
+    stdout: str = ""                                     # captured student prints
+    traceback: str = ""                                  # colored traceback of a code error
+    import_error: Optional[str] = None                   # short "cannot run" summary
+
+    @property
+    def passed(self) -> int:
+        return sum(o.status == PASS for o in self.outcomes)
+
+    @property
+    def failed(self) -> int:
+        return sum(o.status in (FAIL, ERROR) for o in self.outcomes)
+
+    @property
+    def total(self) -> int:
+        return len(self.outcomes)
+
+    @property
+    def has_terminal(self) -> bool:
+        return bool(self.stdout) or bool(self.traceback)
+
+    @property
+    def ok(self) -> bool:
+        return self.import_error is None and self.failed == 0 and not self.undefined
+
+    # ---- plain-text rendering (CLI) -------------------------------------- #
+
+    def to_text(self) -> str:
+        lines = []
+        if self.import_error is not None:
+            lines.append("YOUR CODE CANNOT BE RUN")
+            lines.append("")
+            lines.append(self.import_error)
+        else:
+            for o in self.outcomes:
+                lines.append(f"[{'PASS' if o.status == PASS else 'FAIL'}] {o.name}")
+                if o.status != PASS and o.message:
+                    lines.append(_indent(o.message))
+            if self.undefined:
+                lines += ["", "*" * 57,
+                          "ATTENTION! These functions are not defined (yet):", ""]
+                lines += [f"\t{n}" for n in self.undefined]
+                lines += ["", "They are either misspelled or not written yet.", "*" * 57]
+
+        if self.has_terminal:
+            lines += ["", "----- terminal output -----"]
+            if self.stdout:
+                lines.append(self.stdout.rstrip("\n"))
+            if self.traceback:
+                lines.append(_strip_ansi(self.traceback).rstrip("\n"))
+
+        lines.append("")
+        if self.import_error is not None:
+            lines.append("Fix the error above, then run the tests again.")
+        elif self.ok:
+            lines.append(f"All {self.passed} checks passed. Nice work!")
+        else:
+            bits = f"{self.passed} passed, {self.failed} to fix"
+            if self.undefined:
+                bits += f", {len(self.undefined)} not defined"
+            lines.append(bits + ".")
+        return "\n".join(lines)
+
+
+def _indent(text: str, n: int = 4) -> str:
+    pad = " " * n
+    return "\n".join(pad + line for line in str(text).splitlines())
+
+
+def _strip_ansi(text: str) -> str:
+    import re
+    return re.sub(r"\x1b\[[0-9;]*m", "", text)

im_pytest/resources.py

@@ -0,0 +1,34 @@
+"""Find a project's test file.
+
+Course projects ship as a downloadable ``test_<project>.py`` (plus any data)
+that lands in the student's working folder, so by default we look there. The
+``IM_PROJECT_TESTS`` environment variable (or an explicit ``search`` dir) points
+at a shared course-repo directory instead.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def resolve_test(project: str, search: Optional[str] = None) -> str:
+    fname = f"test_{project}.py"
+    candidates = []
+    if search:
+        candidates.append(Path(search) / fname)
+    env = os.environ.get("IM_PROJECT_TESTS")
+    if env:
+        candidates.append(Path(env) / fname)
+    candidates.append(Path.cwd() / fname)
+    candidates.append(Path.cwd() / "project_tests" / fname)
+    for c in candidates:
+        if c.exists():
+            return str(c)
+    looked = "\n  ".join(str(c) for c in candidates)
+    raise FileNotFoundError(
+        f"Could not find tests for project {project!r} ({fname}).\n"
+        f"Looked in:\n  {looked}\n"
+        f"Download the project's test file into your working folder, or set "
+        f"IM_PROJECT_TESTS to the folder that holds it."
+    )

im_pytest/runner.py

@@ -0,0 +1,197 @@
+"""Run a project's pytest file in-process and return a :class:`Report`.
+
+pytest does the real work; this collects results into a plain object and, crucially,
+separates the two kinds of outcome the widget shows differently:
+
+* an **assertion** about a return value failing → a ``FAIL`` check;
+* the student's **own code raising** (a runtime error, or an import/syntax error
+  that stops the file running) → an ``ERROR`` check *and* a colored traceback +
+  their captured prints in the widget's terminal-output area, just like
+  ``%%exercise``.
+"""
+from __future__ import annotations
+
+import io
+import os
+import types
+from contextlib import redirect_stdout, redirect_stderr
+
+import pytest
+
+from . import plugin as _plugin
+from .report import Report, Outcome, PASS, FAIL, ERROR
+
+try:
+    from IPython import get_ipython
+except Exception:  # pragma: no cover
+    def get_ipython():
+        return None
+
+
+def _clean_name(nodeid: str) -> str:
+    func = nodeid.rsplit("::", 1)[-1]
+    return func[5:] if func.startswith("test_") else func
+
+
+def _assert_message(report) -> str:
+    lr = getattr(report, "longrepr", None)
+    crash = getattr(lr, "reprcrash", None)
+    msg = getattr(crash, "message", None) or (str(lr) if lr else "")
+    lines = [ln for ln in msg.splitlines() if ln.strip()]
+    if len(lines) > 6:
+        lines = lines[:6] + ["... (run `pytest` for the full diff)"]
+    return "\n".join(lines)
+
+
+def _tb_formatter():
+    """The traceback formatter to use, in order of fidelity.
+
+    In a live notebook, the kernel's own ``InteractiveTB`` renders tracebacks
+    exactly as the notebook does for uncaught exceptions — the truest match to
+    ``%%exercise``. Off a kernel (the CLI), fall back to a standalone
+    ``FormattedTB`` on the *neutral* theme (the same theme ``%%exercise`` forces),
+    handling the IPython 9 ``theme_name`` / pre-9 ``color_scheme`` API split.
+    """
+    ip = get_ipython()
+    itb = getattr(ip, "InteractiveTB", None) if ip is not None else None
+    if itb is not None:
+        return itb
+    from IPython.core.ultratb import FormattedTB
+    try:
+        return FormattedTB(mode="Context", theme_name="neutral")   # IPython >= 9
+    except TypeError:  # pragma: no cover - older IPython
+        tb = FormattedTB(mode="Context")
+        try:
+            tb.set_colors("Neutral")
+        except Exception:
+            pass
+        return tb
+
+
+def format_traceback(exc_type, exc_value, exc_tb, student_file=None) -> str:
+    """A colored, student-focused traceback — same look as the ``%%exercise`` widget.
+
+    If ``student_file`` is given, the traceback is sliced to start at the first
+    frame inside the student's own file, so pytest/plumbing frames are hidden.
+    """
+    use_tb = exc_tb
+    if student_file and exc_tb is not None:
+        target = os.path.abspath(student_file)
+        cur = exc_tb
+        while cur is not None:
+            fn = cur.tb_frame.f_code.co_filename
+            if os.path.abspath(fn) == target or fn == student_file:
+                use_tb = cur
+                break
+            cur = cur.tb_next
+    try:
+        tbf = _tb_formatter()
+        return tbf.stb2text(tbf.structured_traceback(exc_type, exc_value, use_tb))
+    except Exception:
+        import traceback
+        return "".join(traceback.format_exception(exc_type, exc_value, use_tb))
+
+
+class _Capture:
+    """Records outcomes, student prints and a code-error traceback for one run."""
+
+    def __init__(self, student_file=None):
+        self.outcomes: list[Outcome] = []
+        self.config = None
+        self.student_file = student_file
+        self.stdout_parts: list[str] = []
+        self.traceback = ""
+        self._done: set[str] = set()
+
+    def pytest_configure(self, config):
+        self.config = config
+
+    def _record(self, name, status, message=""):
+        if name in self._done:
+            return
+        self._done.add(name)
+        self.outcomes.append(Outcome(name, status, message))
+
+    def pytest_exception_interact(self, node, call, report):
+        exc = getattr(call, "excinfo", None)
+        if exc is None:
+            return
+        name = _clean_name(report.nodeid)
+        if issubclass(exc.type, AssertionError):
+            self._record(name, FAIL, _assert_message(report))
+        else:
+            # the student's code raised -> show it in the terminal-output area
+            self._record(name, ERROR, f"raised {exc.type.__name__}: {exc.value}")
+            if not self.traceback:
+                self.traceback = format_traceback(exc.type, exc.value, exc.tb, self.student_file)
+
+    def pytest_runtest_logreport(self, report):
+        if report.when == "call":
+            cap = getattr(report, "capstdout", "")
+            if cap:
+                self.stdout_parts.append(cap)
+            if report.passed:
+                self._record(_clean_name(report.nodeid), PASS)
+
+
+def _run_pytest(test_path, student_file, failfast) -> _Capture:
+    cap = _Capture(student_file=student_file)
+    args = [str(test_path), "-p", "no:cacheprovider", "-q", "--no-header"]
+    if failfast:
+        args.append("-x")
+    sink = io.StringIO()
+    with redirect_stdout(sink), redirect_stderr(sink):
+        pytest.main(args, plugins=[cap])
+    return cap
+
+
+def _build_report(project, cap, pre_stdout="") -> Report:
+    rep = Report(project=project, outcomes=cap.outcomes)
+    if cap.config is not None:
+        rep.undefined = sorted(getattr(cap.config, "_im_undefined", set()) or [])
+    stdout = (pre_stdout or "") + "".join(cap.stdout_parts)
+    rep.stdout = stdout.rstrip("\n")
+    rep.traceback = cap.traceback
+    return rep
+
+
+def run(test_path: str, *, project: str = "", failfast: bool = True) -> Report:
+    """Run ``test_path`` against the student's ``<project>.py`` in the cwd."""
+    project = project or _plugin.student_module_name(os.path.basename(test_path).rsplit(".", 1)[0])
+
+    # Import the student's file *once*, capturing its own prints; a failure here
+    # (syntax error, exception at import time) becomes a friendly "cannot run".
+    pre = io.StringIO()
+    already_injected = project in _plugin._INJECTED
+    try:
+        with redirect_stdout(pre), redirect_stderr(pre):
+            module = _plugin.import_student(project)
+    except Exception as exc:  # noqa: BLE001
+        student_file = os.path.join(os.getcwd(), project + ".py")
+        return Report(
+            project=project,
+            import_error=f"{type(exc).__name__}: {exc}",
+            traceback=format_traceback(type(exc), exc, exc.__traceback__, student_file),
+            stdout=pre.getvalue().rstrip("\n"),
+        )
+
+    # Reuse this exact module inside pytest (no second import, no double prints).
+    if not already_injected:
+        _plugin._INJECTED[project] = module
+    try:
+        cap = _run_pytest(test_path, getattr(module, "__file__", None), failfast)
+    finally:
+        if not already_injected:
+            _plugin._INJECTED.pop(project, None)
+    return _build_report(project, cap, pre_stdout=pre.getvalue())
+
+
+def run_injected(project: str, module: types.ModuleType, test_path: str, *,
+                 pre_stdout: str = "", failfast: bool = True) -> Report:
+    """Run against an in-notebook module built by the ``%%test`` cell magic."""
+    _plugin._INJECTED[project] = module
+    try:
+        cap = _run_pytest(test_path, getattr(module, "__file__", None), failfast)
+    finally:
+        _plugin._INJECTED.pop(project, None)
+    return _build_report(project, cap, pre_stdout=pre_stdout)