modeltest 0.2.0__py3-none-any.whl

modeltest/__init__.py

@@ -0,0 +1,28 @@
+from modeltest.config import register, unregister
+from modeltest.core.base import (
+    ModelSuite,
+    ModelTest,
+    SuiteResult,
+    TestContext,
+    TestResult,
+)
+from modeltest.core.runner import run_suite, run_test
+
+try:
+    from importlib.metadata import version as _metadata_version
+
+    __version__ = _metadata_version("modeltest")
+except Exception:  # noqa: BLE001 - not installed via pip
+    __version__ = "unknown"
+
+__all__ = [
+    "ModelTest",
+    "ModelSuite",
+    "SuiteResult",
+    "TestContext",
+    "TestResult",
+    "run_test",
+    "run_suite",
+    "register",
+    "unregister",
+]

modeltest/cli.py

@@ -0,0 +1,92 @@
+"""Command-line interface: `modeltest validate`."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Optional
+
+from modeltest.core.base import ModelSuite
+
+
+def main(argv: Optional[list] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="modeltest", description="Unit tests for machine learning models."
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    validate = sub.add_parser("validate", help="Run a test suite against a model.")
+    validate.add_argument(
+        "--suite",
+        required=True,
+        help="Path to suite.py (defining `suite`) or suite.yaml (declarative).",
+    )
+    validate.add_argument("--model", required=True, help="Path to model (pickle).")
+    validate.add_argument("--data", required=True, help="Path to validation CSV.")
+    validate.add_argument(
+        "--target", default="target", help="Name of the target column."
+    )
+    validate.add_argument(
+        "--output", default=None, help="Write JUnit XML to this path."
+    )
+    validate.add_argument(
+        "--train-data", default=None, help="Optional training CSV (for drift tests)."
+    )
+    validate.set_defaults(func=_run_validate)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+def _load_suite(path: str) -> "ModelSuite":
+    import os
+
+    ext = os.path.splitext(path)[1].lower()
+    if ext in (".yaml", ".yml"):
+        from modeltest.config import load_suite_yaml
+
+        return load_suite_yaml(path)
+    # default: treat as a Python module exposing `suite`
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("suite_module", os.path.abspath(path))
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod.suite
+
+
+def _run_validate(args: argparse.Namespace) -> int:
+    import os
+
+    import joblib
+    import pandas as pd
+
+    suite: ModelSuite = _load_suite(args.suite)
+    model = joblib.load(os.path.abspath(args.model))
+    df = pd.read_csv(args.data)
+    y = df[args.target]
+    X = df.drop(columns=[args.target])
+
+    X_train = None
+    if args.train_data:
+        train_df = pd.read_csv(args.train_data)
+        X_train = train_df.drop(columns=[args.target])
+
+    result = suite.run(
+        model, X, y, X_train=X_train, model_name=os.path.basename(args.model)
+    )
+
+    if args.output:
+        from modeltest.core.report import to_junit_xml
+
+        out_path = os.path.abspath(args.output)
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+        with open(out_path, "w") as fh:
+            fh.write(to_junit_xml(result))
+
+    print(result.report(style="table"))
+    return 0 if result.passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

modeltest/config.py

@@ -0,0 +1,230 @@
+"""Declarative suite definition: load a ModelSuite from YAML.
+
+The YAML format mirrors the built-in scenarios with a ``type`` and ``params``:
+
+.. code-block:: yaml
+
+    suite:
+      name: "Credit Scoring Model"
+      tests:
+        - type: minimum_accuracy
+          params:
+            threshold: 0.85
+        - type: group_performance
+          params:
+            metric: accuracy
+            threshold: 0.8
+            group_col: "gender"
+        - type: robustness
+          params: {noise_std: 0.01, max_drop: 0.03}
+        - type: data_drift
+          params: {features: [age, income], max_psi: 0.15}
+        - type: equal_opportunity
+          params: {protected_col: "gender", max_diff: 0.1}
+        - type: statistical_parity
+          params: {protected_col: "gender", max_diff: 0.1, min_ratio: 0.8}
+        - type: feature_dominance
+          params: {max_top_share: 0.9}
+        - type: top_features
+          params: {expected_features: [income, age], k: 2}
+        - type: confidence_threshold
+          params: {metric: accuracy, threshold: 0.85, n_boot: 1000, alpha: 0.05}
+        - type: data_invariant
+          params: {expected_columns: [age, income], max_null_ratio: 0.02}
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+
+import yaml
+
+from modeltest.core.base import ModelSuite, ModelTest
+from modeltest.scenarios import (  # type: ignore[attr-defined]
+    ConfidenceThresholdTest,
+    DataDriftTest,
+    DataInvariantTest,
+    EqualOpportunityTest,
+    FeatureDominanceTest,
+    GroupPerformanceTest,
+    KSTest,
+    MinimumAccuracyTest,
+    NoNullTest,
+    RobustnessTest,
+    StatisticalParityTest,
+    TopFeaturesTest,
+)
+
+# Map YAML `type` strings to the scenario class used to build each test.
+_REGISTRY = {
+    "minimum_accuracy": MinimumAccuracyTest,
+    "group_performance": GroupPerformanceTest,
+    "robustness": RobustnessTest,
+    "data_invariant": DataInvariantTest,
+    "no_null": NoNullTest,
+    "data_drift": DataDriftTest,
+    "ks": KSTest,
+    "equal_opportunity": EqualOpportunityTest,
+    "statistical_parity": StatisticalParityTest,
+    "feature_dominance": FeatureDominanceTest,
+    "top_features": TopFeaturesTest,
+    "confidence_threshold": ConfidenceThresholdTest,
+}
+
+
+def register(type_name: str, cls: type) -> None:
+    """Map a YAML ``type`` string to a custom test class.
+
+    Custom tests must subclass :class:`modeltest.ModelTest`. Registering
+    before :func:`load_suite_yaml` lets a suite reference your test by name::
+
+        from modeltest.config import register
+        register("my_error_check", MyErrorCheck)
+
+        # suite.yaml
+        # suite:
+        #   tests:
+        #     - type: my_error_check
+        #       params: {max_errors: 5}
+    """
+    if not (isinstance(cls, type) and issubclass(cls, ModelTest)):
+        raise TypeError(f"register() expects a ModelTest subclass, got {cls!r}")
+    _REGISTRY[type_name] = cls
+
+
+def unregister(type_name: str) -> None:
+    """Remove a previously registered type (built-ins included)."""
+    _REGISTRY.pop(type_name, None)
+
+
+def _resolve_type(type_name: str) -> type:
+    """Look up a test class by registered name or a dotted import path.
+
+    Unknown names may be an import reference of the form ``module.path:Class``
+    or ``module.path.Class`` pointing at a custom test class. This lets a YAML
+    suite load tests defined outside modeltest without a ``register`` call.
+    """
+    if type_name in _REGISTRY:
+        return _REGISTRY[type_name]
+
+    if ":" in type_name:
+        module_path, _, attr = type_name.partition(":")
+        cls = _import_attr(module_path, attr)
+    else:
+        module_path, _, attr = type_name.rpartition(".")
+        cls = _import_attr(module_path, attr) if module_path else None
+
+    if cls is None:
+        raise ValueError(
+            f"Unknown test type {type_name!r}. Known: "
+            f"{', '.join(sorted(_REGISTRY))}. Or use module.path:Class."
+        )
+    return cls
+
+
+def _import_attr(module_path: str, attr: str) -> type:
+    module = _try_import(module_path)
+    cls = getattr(module, attr, None)
+    if cls is None:
+        raise ValueError(f"Module {module_path!r} has no attribute {attr!r}")
+    if not (isinstance(cls, type) and issubclass(cls, ModelTest)):
+        raise TypeError(f"{module_path}.{attr} is not a ModelTest subclass")
+    return cls
+
+
+def _try_import(module_path: str):
+    import sys
+
+    try:
+        return importlib.import_module(module_path)
+    except ImportError:
+        # User-defined test modules are often plain files in the working
+        # directory; make sure CWD is importable so dotted paths "just work"
+        # from the CLI as well as from library code.
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+        try:
+            return importlib.import_module(module_path)
+        except ImportError as exc:
+            raise ValueError(f"Could not import module {module_path!r}: {exc}") from exc
+
+
+def load_suite_yaml(path: str) -> ModelSuite:
+    """Build a :class:`ModelSuite` from a YAML file.
+
+    Raises ``ValueError`` for unknown test types or malformed structure.
+    """
+    with open(path) as fh:
+        doc = yaml.safe_load(fh)
+
+    suite_cfg = doc.get("suite")
+    if not isinstance(suite_cfg, dict):
+        raise ValueError("YAML must contain a top-level `suite:` mapping")
+
+    name = suite_cfg.get("name", "suite")
+    suite = ModelSuite(name=name)
+
+    for raw in suite_cfg.get("tests", []):
+        suite.add_test(_build_test(raw))
+
+    return suite
+
+
+def _build_test(raw) -> object:
+    if not isinstance(raw, dict):
+        raise ValueError(f"Each test must be a mapping, got: {raw!r}")
+
+    type_name = raw.get("type")
+    cls = _resolve_type(type_name)
+
+    params = raw.get("params") or {}
+    if not isinstance(params, dict):
+        raise ValueError(f"`params` for {type_name!r} must be a mapping")
+
+    # Accept clear YAML-friendly aliases for a couple of constructor names.
+    params = {_PARAM_ALIASES.get(k, k): v for k, v in params.items()}
+
+    # Ignore unknown params instead of crashing, but surface typos for hard cases.
+    import inspect
+
+    sig = inspect.signature(cls.__init__)
+    valid = {k for k in sig.parameters if k not in ("self", "args", "kwargs")}
+    unknown = set(params) - valid
+    if unknown:
+        raise ValueError(
+            f"Unknown params for {type_name!r}: {sorted(unknown)}. "
+            f"Valid: {sorted(valid)}"
+        )
+    return cls(**params)
+
+
+_PARAM_ALIASES = {
+    "features": "feature_cols",
+    "columns": "feature_cols",
+    "group": "group_col",
+    "protected": "protected_col",
+}
+
+
+def dump_suite_yaml(suite: ModelSuite, path: str) -> None:
+    """Serialise a suite back to YAML (best-effort; only built-in tests)."""
+    import inspect
+
+    doc = {"suite": {"name": suite.name, "tests": []}}
+    for test in suite.tests:
+        params = {
+            k: v
+            for k, v in vars(test).items()
+            if not k.startswith("_")
+            and k in inspect.signature(type(test).__init__).parameters
+        }
+        doc["suite"]["tests"].append({"type": _inverse_name(test), "params": params})
+    with open(path, "w") as fh:
+        yaml.safe_dump(doc, fh, sort_keys=False)
+
+
+def _inverse_name(test) -> str:
+    inverse = {v: k for k, v in _REGISTRY.items()}
+    return inverse.get(type(test), type(test).__name__)

modeltest/core/__init__.py

@@ -0,0 +1,18 @@
+from modeltest.core.base import (
+    ModelSuite,
+    ModelTest,
+    SuiteResult,
+    TestContext,
+    TestResult,
+)
+from modeltest.core.runner import run_suite, run_test
+
+__all__ = [
+    "ModelTest",
+    "ModelSuite",
+    "SuiteResult",
+    "TestContext",
+    "TestResult",
+    "run_test",
+    "run_suite",
+]

modeltest/core/base.py

@@ -0,0 +1,258 @@
+"""Core primitives: ModelTest, TestContext, TestResult, ModelSuite.
+
+The design mirrors how `pytest` structures tests but is oriented to ML:
+each test receives a rich context (model + data + metadata) instead of a
+bare function signature. A test *passes* by returning normally and *fails*
+by raising an assertion / returning a TestResult with passed=False.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Dict, List, Optional
+
+
+class TestStatus(str, Enum):
+    __test__ = False  # don't let pytest collect this as a test class
+
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    ERROR = "ERROR"
+    SKIPPED = "SKIPPED"
+
+
+@dataclass
+class TestContext:
+    """Everything a test may need at runtime.
+
+    A unified context (rather than a loose `model, X, y` signature) lets tests
+    grow (drift needs train vs val; explainability needs metadata) without
+    breaking the base API.
+    """
+
+    __test__ = False  # don't let pytest collect this as a test class
+
+    model: Any
+    X_val: Any
+    y_val: Any
+    X_train: Optional[Any] = None
+    y_train: Optional[Any] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    cache_predictions: bool = True
+
+    # Per-suite prediction cache shared across every test that runs on this
+    # context (run_suite passes the same instance to all tests).
+    _cache: Optional[Dict[str, Any]] = field(default=None, repr=False)
+
+    _wrapper: Optional[Any] = field(default=None, repr=False)
+
+    @property
+    def model_name(self) -> str:
+        return str(self.metadata.get("model_name", type(self.model).__name__))
+
+    def _wrapped(self) -> Any:
+        """Return a normalized wrapper around ``self.model`` (lazily built)."""
+        if self._wrapper is None:
+            from modeltest.wrappers import wrap
+
+            self._wrapper = wrap(self.model)
+        return self._wrapper
+
+    def predict(self, X: Any = None) -> Any:
+        """Predict with caching.
+
+        Delegates to the model via the framework adapter while transparently
+        dropping non-feature columns (via ``model_features``). When
+        ``cache_predictions`` is on, the result is keyed by a content hash of
+        the input so that multiple tests predicting on the *same data* run the
+        model only once per suite.
+
+        Passing ``X=None`` predicts on ``self.X_val``.
+        """
+        from modeltest.scenarios._utils import model_features
+
+        X = self.X_val if X is None else X
+        wrapped = self._wrapped()
+        X_feat = model_features(wrapped, X)
+
+        if not self.cache_predictions:
+            return wrapped.predict(X_feat)
+
+        if self._cache is None:
+            self._cache = {}
+
+        key = self._fingerprint(X_feat)
+        cached = self._cache.get(key)
+        if cached is not None:
+            return cached
+        pred = wrapped.predict(X_feat)
+        self._cache[key] = pred
+        return pred
+
+    def predict_proba(self, X: Any = None) -> Any:
+        """Probability estimates via the wrapper (or ``None`` if unsupported)."""
+        from modeltest.scenarios._utils import model_features
+
+        X = self.X_val if X is None else X
+        return self._wrapped().predict_proba(model_features(self._wrapped(), X))
+
+    @staticmethod
+    def _fingerprint(X: Any) -> str:
+        """Cheap-ish content key for a validation input."""
+        import hashlib
+
+        if hasattr(X, "columns") and hasattr(X, "values"):
+            # pandas DataFrame / Series: hash column names + row content.
+            try:
+                from pandas.util import hash_pandas_object
+
+                h = hashlib.sha1()
+                h.update("|".join(map(str, X.columns)).encode())
+                h.update(hash_pandas_object(X, index=True).values.tobytes())
+                return h.hexdigest()
+            except Exception:  # noqa: BLE001 - fall back below
+                pass
+        try:
+            import pickle
+
+            return hashlib.sha1(pickle.dumps(X, protocol=4)).hexdigest()
+        except Exception:  # noqa: BLE001
+            return f"id-{id(X)}"
+
+
+@dataclass
+class TestResult:
+    """Outcome of running a single test."""
+
+    __test__ = False  # pytest: an outcome object, not a test hook
+
+    name: str
+    status: TestStatus
+    detail: str = ""
+    duration_ms: float = 0.0
+    metrics: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def passed(self) -> bool:
+        return self.status == TestStatus.PASSED
+
+
+def _now_ms() -> float:
+    import time
+
+    return time.perf_counter() * 1000
+
+
+class ModelTest:
+    """Base class for all model tests.
+
+    Subclasses override `test(self, ctx)`. Raising ``AssertionError`` (or any
+    ``assert`` failure) marks the test as FAILED; raising anything else marks
+    it as ERROR. Returning a :class:`TestResult` lets a test fully control the
+    outcome (useful for warning-only / non-blocking checks).
+    """
+
+    name: Optional[str] = None
+
+    def test(self, ctx: TestContext) -> Any:
+        raise NotImplementedError
+
+    def run(self, ctx: TestContext) -> TestResult:
+        import time
+
+        start = time.perf_counter()
+        status, detail, metrics = TestStatus.PASSED, "", {}
+        try:
+            outcome = self.test(ctx)
+            if isinstance(outcome, TestResult):
+                return outcome
+        except AssertionError as exc:
+            status = TestStatus.FAILED
+            detail = str(exc)
+        except Exception as exc:  # noqa: BLE001 - unknown failures are ERRORs
+            status = TestStatus.ERROR
+            detail = f"{type(exc).__name__}: {exc}"
+        duration = (time.perf_counter() - start) * 1000
+        return TestResult(
+            name=self.name or type(self).__name__,
+            status=status,
+            detail=detail,
+            duration_ms=duration,
+            metrics=metrics,
+        )
+
+
+class ModelSuite:
+    """A collection of tests run together against a model + data."""
+
+    def __init__(self, name: str = "suite", tests: Optional[List[ModelTest]] = None):
+        self.name = name
+        self.tests: List[ModelTest] = list(tests) if tests else []
+
+    def add_test(self, test: ModelTest) -> "ModelSuite":
+        self.tests.append(test)
+        return self
+
+    def add_tests(self, *tests: ModelTest) -> "ModelSuite":
+        self.tests.extend(tests)
+        return self
+
+    def run(
+        self,
+        model: Any,
+        X_val: Any,
+        y_val: Any,
+        X_train: Any = None,
+        y_train: Any = None,
+        **metadata: Any,
+    ) -> "SuiteResult":
+        ctx = TestContext(
+            model=model,
+            X_val=X_val,
+            y_val=y_val,
+            X_train=X_train,
+            y_train=y_train,
+            metadata=metadata or {},
+        )
+        from modeltest.core.runner import run_suite
+
+        return run_suite(self, ctx)
+
+
+@dataclass
+class SuiteResult:
+    """Aggregate outcome of running every test in a suite."""
+
+    suite_name: str
+    results: List[TestResult] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return all(r.passed for r in self.results)
+
+    @property
+    def num_passed(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def num_failed(self) -> int:
+        return sum(1 for r in self.results if not r.passed)
+
+    def report(self, style: str = "table") -> str:
+        from modeltest.core.report import render_report
+
+        return render_report(self, style=style)
+
+
+class _ResultProxy:
+    """Thin dict-like for test authors to attach metrics."""
+
+    pass
+
+
+def assert_metric(
+    actual: float, expected: Any, op: Callable[[float, Any], bool], msg: str
+) -> None:
+    if not op(actual, expected):
+        raise AssertionError(msg)