ml4t-coursework 0.2.0__py3-none-any.whl

ml4t_coursework/__init__.py

@@ -0,0 +1,29 @@
+"""The plumbing every ML4T course notebook runs on.
+
+    !pip install -q ml4t-coursework
+    import ml4t_coursework as mlc
+    mlc.use("foundations")
+
+    from ml4t_coursework import save_component, load_component, append_result, report
+
+The package holds the student's project folder, the components they write and the contracts those
+are checked against, their results log and their submission report. It is named for what it does
+rather than for a course, because more than one course installs it.
+"""
+
+__version__ = "0.2.0"
+
+from .components import catalog, load_component, save_component, source_of, status
+from .contracts import add_source
+from .contracts import get as contract
+from .course import Course, active_course, register_course, use
+from .project import describe, home, setup
+from .report import report
+from .results import append_result, latest, results
+
+__all__ = [
+    "save_component", "load_component", "append_result", "report",
+    "setup", "home", "describe", "status", "catalog", "contract", "results", "latest",
+    "source_of", "use", "Course", "register_course", "active_course", "add_source",
+    "__version__",
+]

ml4t_coursework/checks.py

@@ -0,0 +1,221 @@
+"""The five checks every component contract carries, and the failure they raise.
+
+`certification.md` § 2 is the specification. The checks assert properties, never equality with a
+reference output: two correct fold splitters legitimately differ and an equality check would fail
+correct work and teach copying. The reference delta is the one exception and it is reported, not
+enforced.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+import numpy as np
+import pandas as pd
+
+
+class Failure(Exception):
+    """A contract check that did not pass.
+
+    Every failure names the expectation and the observed value, because "expected 12 folds, got 9"
+    is feedback and a bare AssertionError is not.
+    """
+
+    def __init__(self, check: str, expected: str, observed: str, hint: str = "") -> None:
+        self.check = check
+        self.expected = expected
+        self.observed = observed
+        self.hint = hint
+        message = f"{check}: expected {expected}, got {observed}"
+        if hint:
+            message = f"{message}\n  {hint}"
+        super().__init__(message)
+
+
+def require(condition: bool, check: str, expected: str, observed: str, hint: str = "") -> None:
+    if not condition:
+        raise Failure(check, expected, observed, hint)
+
+
+@dataclass
+class CheckResult:
+    name: str
+    passed: bool
+    detail: str = ""
+
+    def line(self) -> str:
+        mark = "pass" if self.passed else "FAIL"
+        return f"  [{mark}] {self.name}{': ' + self.detail if self.detail else ''}"
+
+
+@dataclass
+class Conformance:
+    component: str
+    conformant: bool
+    checks: list[CheckResult] = field(default_factory=list)
+    delta: str = ""
+    stamped_at: str = ""
+    helper_version: str = ""
+    unit: str = ""
+
+    def __str__(self) -> str:
+        head = f"{self.component}: {'conformant' if self.conformant else 'NOT conformant'}"
+        body = "\n".join(c.line() for c in self.checks)
+        tail = f"\n  [note] reference delta: {self.delta}" if self.delta else ""
+        return f"{head}\n{body}{tail}"
+
+    def to_dict(self) -> dict:
+        return {
+            "component": self.component,
+            "unit": self.unit,
+            "conformant": self.conformant,
+            "checks": [{"name": c.name, "passed": c.passed, "detail": c.detail} for c in self.checks],
+            "reference_delta": self.delta,
+            "stamped_at": self.stamped_at,
+            "helper_version": self.helper_version,
+        }
+
+
+# --- comparison helpers, shared by determinism, the leakage probe and the delta ---------------
+
+
+def same(left: Any, right: Any, tol: float = 1e-12) -> bool:
+    """Value equality that understands the shapes components actually return."""
+    if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
+        if left.shape != right.shape or not left.index.equals(right.index):
+            return False
+        if list(left.columns) != list(right.columns):
+            return False
+        return bool(np.allclose(left.to_numpy(dtype=float), right.to_numpy(dtype=float),
+                                atol=tol, rtol=0, equal_nan=True))
+    if isinstance(left, pd.Series) and isinstance(right, pd.Series):
+        if not left.index.equals(right.index):
+            return False
+        return bool(np.allclose(left.to_numpy(dtype=float), right.to_numpy(dtype=float),
+                                atol=tol, rtol=0, equal_nan=True))
+    if isinstance(left, (list, tuple)) and isinstance(right, (list, tuple)):
+        return len(left) == len(right) and all(same(a, b, tol) for a, b in zip(left, right))
+    if isinstance(left, pd.Index) and isinstance(right, pd.Index):
+        return left.equals(right)
+    if isinstance(left, np.ndarray) and isinstance(right, np.ndarray):
+        return left.shape == right.shape and bool(np.allclose(left, right, atol=tol, equal_nan=True))
+    return bool(left == right)
+
+
+def overlap(left: Any, right: Any) -> tuple[Any, Any]:
+    """Restrict two indexed objects to the rows they share, in the same order."""
+    if isinstance(left, (pd.Series, pd.DataFrame)) and isinstance(right, (pd.Series, pd.DataFrame)):
+        shared = left.index.intersection(right.index)
+        return left.loc[shared], right.loc[shared]
+    return left, right
+
+
+def describe_delta(student: Any, reference: Any) -> str:
+    """One line on how far the student's output sits from the shipped reference's.
+
+    Reported and never enforced. Where a design choice legitimately differs the delta is
+    information; only the properties gate.
+    """
+    try:
+        a, b = overlap(student, reference)
+        if isinstance(a, (pd.Series, pd.DataFrame)) and isinstance(b, (pd.Series, pd.DataFrame)):
+            if len(a) == 0:
+                return "no overlapping rows, so nothing to compare"
+            if isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame):
+                cols = a.columns.intersection(b.columns)
+                if len(cols) == 0:
+                    return "no overlapping columns, so nothing to compare"
+                a, b = a[cols], b[cols]
+            diff = np.abs(np.asarray(a, dtype=float) - np.asarray(b, dtype=float))
+            scale = np.nanmean(np.abs(np.asarray(b, dtype=float)))
+            mean = np.nanmean(diff)
+            rel = f", about {mean / scale:.1%} of the reference's own scale" if scale else ""
+            return f"mean absolute difference {mean:.6g} over {len(a)} rows{rel}"
+        if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
+            return f"{len(a)} against the reference's {len(b)}"
+        if isinstance(a, dict) and isinstance(b, dict):
+            differing = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
+            return "identical to the reference" if not differing else f"differs on {', '.join(differing)}"
+        return "identical to the reference" if same(a, b) else f"{a!r} against the reference's {b!r}"
+    except Exception as exc:  # a delta is information; it must never break a save
+        return f"not computable ({type(exc).__name__})"
+
+
+# --- the four gating checks --------------------------------------------------------------------
+
+
+def run_interface(contract, obj) -> CheckResult:
+    contract.interface(obj)
+    return CheckResult("interface", True, contract.interface_detail)
+
+
+def run_determinism(contract, obj) -> CheckResult:
+    first = contract.probe(obj)
+    second = contract.probe(obj)
+    require(
+        same(first, second),
+        "determinism",
+        "the same output from the same input twice",
+        "two different outputs",
+        "Something in the component is drawing on state that changes between calls - an unseeded "
+        "random draw, a mutable default, or a value read from the clock.",
+    )
+    return CheckResult("determinism", True, "same input twice, same output")
+
+
+def run_leakage(contract, obj) -> CheckResult:
+    if contract.leakage is None:
+        return CheckResult("leakage probe", True, contract.leakage_note)
+    detail = contract.leakage(obj)
+    return CheckResult("leakage probe", True, detail or contract.leakage_note)
+
+
+def run_invariants(contract, obj) -> list[CheckResult]:
+    results = []
+    for label, fn in contract.invariants:
+        detail = fn(obj)
+        results.append(CheckResult(label, True, detail or ""))
+    return results
+
+
+def evaluate(contract, obj) -> Conformance:
+    """Run all five checks, stopping the gating ones at the first failure."""
+    result = Conformance(component=contract.name, unit=contract.units[0], conformant=True)
+    stages: list[Callable[[], Any]] = [
+        lambda: [run_interface(contract, obj)],
+        lambda: [run_leakage(contract, obj)],
+        lambda: [run_determinism(contract, obj)],
+        lambda: run_invariants(contract, obj),
+    ]
+    names = ["interface", "leakage probe", "determinism", "domain invariants"]
+    for name, stage in zip(names, stages):
+        try:
+            result.checks.extend(stage())
+        except Failure as failure:
+            result.checks.append(CheckResult(failure.check, False, str(failure).split(": ", 1)[1]))
+            result.conformant = False
+            break
+        except NameError as exc:
+            missing = str(exc).split("'")[1] if "'" in str(exc) else "something"
+            result.checks.append(CheckResult(name, False, (
+                f"the component calls {missing!r}, which is not part of it. "
+                f"Colab does not keep your session, so a later notebook loads this file on its own "
+                f"and {missing!r} will not be there. Either move it inside the component, or pass "
+                f"it along: save_component(..., also=[{missing}]) for a function, "
+                f"include={{{missing!r}: ...}} for a value."
+            )))
+            result.conformant = False
+            break
+        except Exception as exc:
+            result.checks.append(
+                CheckResult(name, False, f"the check could not run: {type(exc).__name__}: {exc}")
+            )
+            result.conformant = False
+            break
+    if result.conformant:
+        try:
+            result.delta = describe_delta(contract.probe(obj), contract.probe(contract.reference()))
+        except Exception as exc:
+            result.delta = f"not computable ({type(exc).__name__})"
+    return result

ml4t_coursework/components.py

@@ -0,0 +1,204 @@
+"""`save_component` and `load_component`: how a component is checked, stamped and recovered.
+
+What persists between units is the student's *code*, not intermediate data. A notebook defines its
+component from scratch and writes it here; a later pipeline notebook loads it, or falls back to the
+shipped reference and says so. Restart-and-run-all works on any unit, in any order, on a cold
+session, which is the property this module exists to protect.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import inspect
+import json
+import textwrap
+from typing import Any, Callable, Sequence
+
+from . import contracts, project
+from .checks import Conformance, evaluate
+
+__version_marker__ = "components"
+
+_PREAMBLE = (
+    "# Saved by ml4t-coursework. This is your own code, exactly as you wrote it.\n"
+    "import numpy as np\n"
+    "import pandas as pd\n\n"
+)
+
+
+def _version() -> str:
+    from . import __version__
+
+    return __version__
+
+
+def _now() -> str:
+    return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
+
+
+def _source_of(obj: Any, name: str) -> str:
+    try:
+        return textwrap.dedent(inspect.getsource(obj))
+    except (OSError, TypeError) as exc:
+        raise ValueError(
+            f"Could not read the source of the object you passed for {name!r}.\n"
+            f"  Define it as a function or a class in a notebook cell and pass the object itself, "
+            f"not a lambda, a partial or an instance.\n"
+            f"  ({type(exc).__name__}: {exc})"
+        ) from exc
+
+
+def _rebuild(source: str, symbol: str, name: str) -> Any:
+    """Execute the saved source in a fresh namespace and return the object.
+
+    This is deliberately done before the checks run. What gets validated has to be what a later
+    notebook will actually load, and a component that only works because of something else in the
+    student's session would pass in place and fail on a cold kernel.
+    """
+    namespace: dict[str, Any] = {}
+    import numpy as np
+    import pandas as pd
+
+    namespace.update({"np": np, "pd": pd, "numpy": np, "pandas": pd})
+    try:
+        exec(compile(_PREAMBLE + source, f"<{name}>", "exec"), namespace)
+    except NameError as exc:
+        missing = str(exc).split("'")[1] if "'" in str(exc) else "something"
+        raise ValueError(
+            f"Your {name} refers to {missing!r}, which is not part of it.\n"
+            f"  Colab does not keep your session, so a later notebook loads this file on its own "
+            f"and {missing!r} will not be there.\n"
+            f"  Either move it inside the component, or pass it along: "
+            f"save_component({name!r}, {symbol}, also=[{missing}])"
+        ) from exc
+    if symbol not in namespace:
+        raise ValueError(
+            f"The saved source does not define {symbol!r}.\n"
+            f"  Name the function or class you pass, and pass it by that name."
+        )
+    return namespace[symbol]
+
+
+def save_component(
+    name: str,
+    obj: Any,
+    also: Sequence[Callable] = (),
+    include: dict[str, Any] | None = None,
+    quiet: bool = False,
+) -> Conformance:
+    """Check a component against its contract, stamp the verdict, and write it to the project.
+
+    `also` carries any function you defined in another cell that the component calls; `include`
+    carries any plain value it reads, as `{name: value}`. A component has to stand on its own,
+    because a later notebook loads this file on a cold session with nothing else in scope.
+    """
+    contract = contracts.get(name)
+    folder = project.components_dir()
+
+    if contract.kind == "config":
+        if not isinstance(obj, dict):
+            raise ValueError(
+                f"{name} is a recorded choice, not a function.\n"
+                f"  Pass a dictionary. Expected keys: {contract.interface_detail}\n"
+                f"  You passed a {type(obj).__name__}."
+            )
+        payload = dict(obj)
+        checked: Any = payload
+        body = json.dumps(payload, indent=2, default=str)
+        target = folder / f"{name}.json"
+        symbol = name
+    else:
+        symbol = getattr(obj, "__name__", None)
+        if not symbol:
+            raise ValueError(
+                f"{name}: pass the function or class itself, by name, not an instance or a lambda."
+            )
+        parts = [f"{key} = {value!r}" for key, value in (include or {}).items()]
+        parts += [_source_of(helper, name) for helper in also]
+        parts.append(_source_of(obj, name))
+        body = "\n\n".join(parts)
+        checked = _rebuild(body, symbol, name)
+        target = folder / f"{name}.py"
+
+    result = evaluate(contract, checked)
+    result.stamped_at = _now()
+    result.helper_version = _version()
+
+    target.write_text(_PREAMBLE + body if contract.kind == "callable" else body)
+    meta = result.to_dict() | {"symbol": symbol, "file": target.name, "kind": contract.kind}
+    (folder / f"{name}.meta.json").write_text(json.dumps(meta, indent=2))
+
+    if not quiet:
+        print(result)
+        if result.conformant:
+            print(f"  saved to {target}")
+        else:
+            print(f"  saved to {target}, and it is recorded as not conformant.")
+            print("  Nothing is blocked: fix it whenever you like, and any pipeline notebook you "
+                  "run meanwhile uses the shipped reference for this step.")
+    return result
+
+
+def _meta(name: str) -> dict | None:
+    path = project.components_dir(create=False) / f"{name}.meta.json"
+    if not path.is_file():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except json.JSONDecodeError:
+        return None
+
+
+def source_of(name: str) -> str:
+    """`'yours'` or `'reference'` - what `load_component` would return right now."""
+    meta = _meta(name)
+    return "yours" if meta and meta.get("conformant") else "reference"
+
+
+def load_component(name: str, quiet: bool = False) -> Any:
+    """Return the student's component when it is conformant, the shipped reference otherwise.
+
+    It always says which. This is what stops a wrong Part 4 from blocking Part 7.
+    """
+    contract = contracts.get(name)
+    meta = _meta(name)
+    if meta and meta.get("conformant"):
+        folder = project.components_dir(create=False)
+        path = folder / meta["file"]
+        try:
+            if contract.kind == "config":
+                obj = json.loads(path.read_text())
+            else:
+                obj = _rebuild(path.read_text(), meta["symbol"], name)
+            if not quiet:
+                stamped = meta.get("stamped_at", "")[:10]
+                print(f"{name}: using your implementation (checked {stamped})")
+            return obj
+        except Exception as exc:
+            if not quiet:
+                print(f"{name}: your saved version would not load ({type(exc).__name__}), so this "
+                      f"run uses the shipped reference.")
+    elif not quiet:
+        reason = "you have not written it yet" if meta is None else "yours is recorded as not conformant"
+        print(f"{name}: using the shipped reference, because {reason}.")
+    return contract.reference()
+
+
+def status() -> str:
+    """Every component the course asks for, and where each one stands."""
+    lines = []
+    for name, contract in sorted(contracts.load_all().items()):
+        meta = _meta(name)
+        if meta is None:
+            state = "not written yet"
+        elif meta.get("conformant"):
+            state = f"conformant, {meta.get('stamped_at', '')[:10]}"
+        else:
+            failed = [c["name"] for c in meta.get("checks", []) if not c["passed"]]
+            state = f"NOT conformant ({', '.join(failed) or 'unknown'})"
+        lines.append(f"  {name:<20} unit {contract.units[0]:<5} {state}")
+    return "components in your project folder:\n" + "\n".join(lines)
+
+
+def catalog() -> str:
+    return contracts.catalog()

ml4t_coursework/contracts.py

@@ -0,0 +1,101 @@
+"""What a component is, and the registry of the twenty the course asks for.
+
+One `Contract` per component, declared beside its reference implementation in `reference/`. The
+contract is authored *from* the reference, which is what guarantees it is passable, and every
+reference is tested against its own contract plus a deliberately broken variant that must fail it.
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from dataclasses import dataclass
+from types import ModuleType
+from typing import Any, Callable
+
+CONTRACTS: dict[str, "Contract"] = {}
+
+_SOURCES: list[str] = [f"{__package__}.reference"]
+_LOADED: set[str] = set()
+
+
+@dataclass
+class Contract:
+    name: str
+    kind: str
+    units: tuple[str, ...]
+    summary: str
+    probe: Callable[[Any], Any]
+    interface: Callable[[Any], None]
+    reference: Callable[[], Any]
+    interface_detail: str = ""
+    invariants: tuple[tuple[str, Callable[[Any], str | None]], ...] = ()
+    leakage: Callable[[Any], str | None] | None = None
+    leakage_note: str = "not applicable to this component, and here is why"
+    dependencies: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if self.kind not in {"callable", "config"}:
+            raise ValueError(f"{self.name}: kind must be 'callable' or 'config', got {self.kind!r}")
+
+    def describe(self) -> str:
+        lines = [f"{self.name} ({self.kind}, written in unit {' and '.join(self.units)})",
+                 f"  {self.summary}",
+                 f"  interface: {self.interface_detail}",
+                 f"  leakage probe: {self.leakage_note}"]
+        for label, _ in self.invariants:
+            lines.append(f"  invariant: {label}")
+        return "\n".join(lines)
+
+
+def register(contract: Contract) -> Contract:
+    existing = CONTRACTS.get(contract.name)
+    if existing is not None and existing is not contract:
+        raise ValueError(
+            f"Two different contracts are both called {contract.name!r}.\n"
+            f"  Already registered from: {existing.reference.__module__}\n"
+            f"  Now registering from:    {contract.reference.__module__}\n"
+            f"  Component names are shared across every registered source, so pick another."
+        )
+    CONTRACTS[contract.name] = contract
+    return contract
+
+
+def add_source(package: str | ModuleType) -> None:
+    """Add a package of contract modules to discovery. A course outside this one calls this."""
+    name = package if isinstance(package, str) else package.__name__
+    if name not in _SOURCES:
+        _SOURCES.append(name)
+
+
+def load_all() -> dict[str, Contract]:
+    """Import every module in every registered source package, each of which registers a contract.
+
+    Tracked per source rather than by whether the registry is empty: a caller that pre-registers
+    one contract of its own must not suppress the import of every other source.
+    """
+    for source in list(_SOURCES):
+        if source in _LOADED:
+            continue
+        _LOADED.add(source)
+        package = importlib.import_module(source)
+        for module in pkgutil.iter_modules(package.__path__):
+            if not module.name.startswith("_"):
+                importlib.import_module(f"{package.__name__}.{module.name}")
+    return CONTRACTS
+
+
+def get(name: str) -> Contract:
+    contracts = load_all()
+    if name not in contracts:
+        known = ", ".join(sorted(contracts))
+        raise KeyError(
+            f"There is no component called {name!r} in this course.\n"
+            f"  The ones there are: {known}\n"
+            f"  Check the spelling against the name the unit's notebook asked you to save."
+        )
+    return contracts[name]
+
+
+def catalog() -> str:
+    return "\n\n".join(c.describe() for _, c in sorted(load_all().items()))

ml4t_coursework/course.py

@@ -0,0 +1,86 @@
+"""Which course a notebook belongs to, and the few facts that differ between them.
+
+Almost everything here is shared: the checks, the contract machinery, the results columns, the
+report. Three things are not, and they are the three a package named after one course would have
+gotten wrong - the project folder on the student's Drive, the environment override that points at
+it, and the list of stage names a results row may carry. They live on a `Course` and a notebook
+selects one in its opening cell.
+
+There is deliberately no default. A notebook that never says which course it belongs to would
+otherwise write its components into another course's folder, and the failure would surface weeks
+later as a component that loads but does the wrong thing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+COURSES: dict[str, "Course"] = {}
+_active: str | None = None
+
+
+@dataclass(frozen=True)
+class Course:
+    key: str
+    title: str
+    folder: str
+    env_var: str
+    stages: tuple[str, ...]
+    terminal_stages: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        unknown = [s for s in self.terminal_stages if s not in self.stages]
+        if unknown:
+            raise ValueError(
+                f"{self.key}: terminal stages must also be stages, and "
+                f"{', '.join(unknown)} is not."
+            )
+
+
+def register_course(course: Course, replace: bool = False) -> Course:
+    existing = COURSES.get(course.key)
+    if existing is not None and existing != course and not replace:
+        raise ValueError(
+            f"A different course is already registered as {course.key!r}.\n"
+            f"  Registered: {existing}\n"
+            f"  Pass replace=True if you meant to change it."
+        )
+    COURSES[course.key] = course
+    return course
+
+
+def use(key: str) -> Course:
+    """Select the course this session belongs to. Every notebook calls this in its opening cell."""
+    global _active
+    if key not in COURSES:
+        known = ", ".join(sorted(COURSES)) or "none yet"
+        raise KeyError(
+            f"There is no course registered as {key!r}.\n"
+            f"  Registered: {known}\n"
+            f"  A course registers itself with ml4t_coursework.register_course()."
+        )
+    _active = key
+    return COURSES[key]
+
+
+def active_course() -> Course:
+    if _active is None:
+        known = ", ".join(repr(k) for k in sorted(COURSES)) or "none yet"
+        raise RuntimeError(
+            "This session has not said which course it belongs to, so there is no project "
+            "folder to read or write.\n"
+            f"  Add ml4t_coursework.use(...) to the notebook's opening cell, one of: {known}.\n"
+            "  The setup notebook does this and so does every unit notebook."
+        )
+    return COURSES[_active]
+
+
+register_course(Course(
+    key="foundations",
+    title="ML4T: Foundations",
+    folder="ml4t-foundations",
+    env_var="ML4T_FOUNDATIONS_HOME",
+    stages=("baseline", "v0", "part_02", "part_04", "part_05", "part_06", "part_07", "part_08",
+            "final"),
+    terminal_stages=("baseline", "v0", "final"),
+))