evalsuite-python 0.1.0a1__py3-none-any.whl

evalsuite/core/__init__.py

@@ -0,0 +1 @@
+"""Core infrastructure: exceptions, validation, results, registry and evaluation context."""

evalsuite/core/context.py

@@ -0,0 +1,181 @@
+"""EvaluationContext: validate inputs once and cache shared intermediate results.
+
+Every classification metric derives from per-class counts (TP, FP, FN, TN). The context computes the
+confusion matrix once; ``evaluate()`` passes one context to all requested metrics, so ten metrics cost one
+pass over the data.
+"""
+
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Any, Optional
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .exceptions import InputValidationError, UnsupportedTaskError
+from .types import ArrayLike, FloatArray
+from .validation import (
+    TargetType,
+    check_consistent_length,
+    check_finite,
+    resolve_labels,
+    target_type,
+    to_numpy,
+    validate_probabilities,
+    validate_sample_weight,
+)
+
+__all__ = ["ClassificationContext"]
+
+
+class ClassificationContext:
+    """Validated classification inputs plus cached counts.
+
+    For binary and multiclass targets, ``labels`` defines the class order used by every per-class output and
+    by the columns of 2-D ``y_prob``. For multilabel targets, labels are column indices.
+    """
+
+    def __init__(
+        self,
+        y_true: ArrayLike,
+        y_pred: Optional[ArrayLike] = None,
+        *,
+        y_prob: Optional[ArrayLike] = None,
+        labels: Optional[ArrayLike] = None,
+        sample_weight: Optional[ArrayLike] = None,
+    ) -> None:
+        yt = to_numpy(y_true, "y_true", allow_2d=True)
+        yp = None if y_pred is None else to_numpy(y_pred, "y_pred", allow_2d=True)
+        check_finite(yt, "y_true")
+        if yp is not None:
+            check_finite(yp, "y_pred")
+        self.n = check_consistent_length(y_true=yt, y_pred=yp)
+        kind = target_type(yt)
+        if kind == "continuous":
+            raise UnsupportedTaskError(
+                "y_true contains non-integer numbers, which looks like a regression target. "
+                'Use evalsuite.regression metrics or evaluate(..., task="regression").'
+            )
+        if yp is not None and yt.ndim != yp.ndim:
+            raise InputValidationError(
+                f"y_true has {yt.ndim} dimension(s) but y_pred has {yp.ndim}. Multilabel targets need both as "
+                "indicator matrices; single-label targets need both as 1-D label arrays."
+            )
+        if kind == "multilabel" and yp is not None:
+            if yp.shape[1] != yt.shape[1]:
+                raise InputValidationError(f"y_true has {yt.shape[1]} label columns but y_pred has {yp.shape[1]}.")
+            if not np.isin(yp, (0, 1)).all():
+                raise InputValidationError("Multilabel y_pred must be a 0/1 indicator matrix.")
+        self.target_type: TargetType = kind
+        self.y_true = yt
+        self.y_pred = yp
+        self.sample_weight = validate_sample_weight(sample_weight, self.n)
+        if kind == "multilabel":
+            self.labels: NDArray[Any] = np.arange(yt.shape[1])
+        else:
+            self.labels = resolve_labels(yt, yp, labels)
+            if labels is not None:
+                observed = np.unique(yt if yp is None else np.concatenate([yt, yp]))
+                missing = np.setdiff1d(observed, self.labels)
+                if missing.size:
+                    raise InputValidationError(
+                        f"y_true/y_pred contain label(s) {missing.tolist()} that are not in labels. "
+                        "Include every label that occurs, or filter the data first."
+                    )
+            if kind == "binary" and self.labels.shape[0] > 2:
+                self.target_type = "multiclass"
+        self._y_prob_raw = y_prob
+
+    # ---- encodings -------------------------------------------------------------------------------
+    @cached_property
+    def _index(self) -> dict[Any, int]:
+        return {lab.item() if hasattr(lab, "item") else lab: i for i, lab in enumerate(self.labels)}
+
+    def encode(self, y: NDArray[Any]) -> NDArray[np.int64]:
+        sorted_labels = np.all(self.labels[:-1] <= self.labels[1:]) if self.labels.dtype != object else False
+        if sorted_labels:
+            return np.searchsorted(self.labels, y).astype(np.int64)
+        idx = self._index
+        return np.fromiter((idx[v.item() if hasattr(v, "item") else v] for v in y), dtype=np.int64, count=len(y))
+
+    @cached_property
+    def true_idx(self) -> NDArray[np.int64]:
+        return self.encode(self.y_true)
+
+    @cached_property
+    def pred_idx(self) -> NDArray[np.int64]:
+        if self.y_pred is None:
+            raise InputValidationError("This metric needs y_pred (predicted labels).")
+        return self.encode(self.y_pred)
+
+    @cached_property
+    def weights(self) -> FloatArray:
+        return np.ones(self.n) if self.sample_weight is None else self.sample_weight
+
+    # ---- cached counts ---------------------------------------------------------------------------
+    @cached_property
+    def confusion_matrix(self) -> FloatArray:
+        """Rows: true labels; columns: predicted labels (weighted counts). Single-label targets only."""
+        if self.target_type == "multilabel":
+            raise UnsupportedTaskError(
+                "A single confusion matrix is not defined for multilabel targets; use multilabel_counts."
+            )
+        k = self.labels.shape[0]
+        flat = self.true_idx * k + self.pred_idx
+        return np.bincount(flat, weights=self.weights, minlength=k * k).reshape(k, k).astype(np.float64)
+
+    @cached_property
+    def counts(self) -> dict[str, FloatArray]:
+        """Per-class (or per-label) weighted tp, fp, fn, tn and support."""
+        if self.target_type == "multilabel":
+            if self.y_pred is None:
+                raise InputValidationError("This metric needs y_pred (predicted labels).")
+            w = self.weights[:, None]
+            t = self.y_true.astype(bool)
+            p = self.y_pred.astype(bool)
+            tp = ((t & p) * w).sum(0)
+            fp = ((~t & p) * w).sum(0)
+            fn = ((t & ~p) * w).sum(0)
+            tn = ((~t & ~p) * w).sum(0)
+        else:
+            cm = self.confusion_matrix
+            tp = np.diag(cm).copy()
+            fp = cm.sum(0) - tp
+            fn = cm.sum(1) - tp
+            tn = cm.sum() - tp - fp - fn
+        return {"tp": tp, "fp": fp, "fn": fn, "tn": tn, "support": tp + fn}
+
+    @cached_property
+    def total_weight(self) -> float:
+        return float(self.weights.sum())
+
+    # ---- probabilities ---------------------------------------------------------------------------
+    @cached_property
+    def y_prob(self) -> FloatArray:
+        if self._y_prob_raw is None:
+            raise InputValidationError(
+                "This metric needs predicted probabilities (y_prob): P(positive class) for binary tasks, "
+                "or one column per class in label order for multiclass tasks."
+            )
+        k = self.labels.shape[0]
+        p = validate_probabilities(
+            self._y_prob_raw,
+            self.n,
+            n_classes=None if self.target_type == "binary" else k,
+            rows_sum_to_one=self.target_type == "multiclass",
+            unit="label" if self.target_type == "multilabel" else "class",
+        )
+        if self.target_type == "binary" and p.ndim == 2:
+            if p.shape[1] != 2:
+                raise InputValidationError(
+                    f"For binary tasks y_prob must be P(positive) or two columns; received {p.shape[1]} columns."
+                )
+            p = p[:, 1]
+        if self.target_type == "multiclass" and p.ndim == 1:
+            raise InputValidationError(
+                f"Multiclass tasks need y_prob with one column per class ({k} columns, in label order)."
+            )
+        if self.target_type == "multilabel" and (p.ndim != 2 or p.shape[1] != self.y_true.shape[1]):
+            raise InputValidationError("Multilabel y_prob must have one probability column per label.")
+        return p

evalsuite/core/exceptions.py

@@ -0,0 +1,52 @@
+"""Exception hierarchy.
+
+Every message says what failed, why, and how to fix it.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "EvalSuiteError",
+    "InputValidationError",
+    "MetricInputError",
+    "OptionalDependencyError",
+    "StatisticalTestError",
+    "UndefinedMetricWarning",
+    "UnsupportedTaskError",
+]
+
+
+class EvalSuiteError(Exception):
+    """Base class for all EvalSuite errors."""
+
+
+class InputValidationError(EvalSuiteError, ValueError):
+    """Inputs have the wrong shape, length, type or values."""
+
+
+class MetricInputError(EvalSuiteError, ValueError):
+    """Inputs are valid arrays but outside the domain a metric is defined on."""
+
+
+class UnsupportedTaskError(EvalSuiteError, ValueError):
+    """The requested task or averaging mode is not supported for this metric."""
+
+
+class OptionalDependencyError(EvalSuiteError, ImportError):
+    """A feature needs an optional dependency that is not installed."""
+
+    def __init__(self, package: str, extra: str, feature: str) -> None:
+        super().__init__(
+            f"{feature} requires the optional dependency '{package}'. "
+            f'Install it with: pip install "evalsuite-python[{extra}]"'
+        )
+        self.package = package
+        self.extra = extra
+
+
+class StatisticalTestError(EvalSuiteError, ValueError):
+    """A statistical procedure cannot be applied to the given data."""
+
+
+class UndefinedMetricWarning(UserWarning):
+    """A metric is undefined for the input (for example a zero denominator)."""

evalsuite/core/registry.py

@@ -0,0 +1,90 @@
+"""Metric registry: programmatic discovery of every metric and its documentation."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import asdict, dataclass, field
+from typing import Any, Optional, TypeVar
+
+__all__ = ["MetricInfo", "list_metrics", "metric_info", "register"]
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+@dataclass(frozen=True)
+class MetricInfo:
+    """Documentation for one metric. ``id`` is ``"<category>.<name>"``, e.g. ``"classification.f1"``."""
+
+    id: str
+    name: str
+    category: str
+    task: str
+    definition: str
+    formula: str
+    range: str
+    input_requirements: tuple[str, ...] = ()
+    references: tuple[str, ...] = ()
+    higher_is_better: Optional[bool] = True
+    function: Optional[Callable[..., Any]] = field(default=None, repr=False, compare=False)
+
+    def to_dict(self) -> dict[str, Any]:
+        out = asdict(self)
+        out.pop("function", None)
+        out["input_requirements"] = list(self.input_requirements)
+        out["references"] = list(self.references)
+        out["api"] = f"evalsuite.{self.id}"
+        return out
+
+
+_REGISTRY: dict[str, MetricInfo] = {}
+
+
+def register(
+    *,
+    category: str,
+    task: str,
+    name: str,
+    definition: str,
+    formula: str,
+    range: str,
+    input_requirements: tuple[str, ...] = (),
+    references: tuple[str, ...] = (),
+    higher_is_better: Optional[bool] = True,
+) -> Callable[[F], F]:
+    """Decorator registering a public metric function under ``<category>.<function name>``."""
+
+    def deco(fn: F) -> F:
+        metric_id = f"{category}.{fn.__name__}"
+        if metric_id in _REGISTRY:
+            raise RuntimeError(f"Metric {metric_id} is registered twice.")
+        _REGISTRY[metric_id] = MetricInfo(
+            id=metric_id,
+            name=name,
+            category=category,
+            task=task,
+            definition=definition,
+            formula=formula,
+            range=range,
+            input_requirements=input_requirements,
+            references=references,
+            higher_is_better=higher_is_better,
+            function=fn,
+        )
+        return fn
+
+    return deco
+
+
+def list_metrics(category: Optional[str] = None) -> list[str]:
+    """Sorted metric ids, optionally filtered by category (``"classification"``, ``"regression"``)."""
+    return sorted(k for k, v in _REGISTRY.items() if category is None or v.category == category)
+
+
+def metric_info(metric_id: str) -> MetricInfo:
+    """Documentation for one metric, e.g. ``metric_info("classification.mcc")``."""
+    try:
+        return _REGISTRY[metric_id]
+    except KeyError:
+        close = [k for k in _REGISTRY if metric_id.split(".")[-1] in k]
+        hint = f" Did you mean: {', '.join(close)}?" if close else " Use evalsuite.list_metrics() to see all ids."
+        raise KeyError(f"Unknown metric '{metric_id}'.{hint}") from None

evalsuite/core/result.py

@@ -0,0 +1,277 @@
+"""Immutable result objects with portable exports (dict, JSON, DataFrame, Markdown, LaTeX).
+
+JSON uses the standard library only; NaN and infinity become ``null`` so the output is valid JSON.
+Pickle is never used.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import numpy as np
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+__all__ = ["EvaluationResult", "MetricResult"]
+
+Number = Union[float, int]
+
+
+def _json_safe(value: Any) -> Any:
+    if isinstance(value, (np.floating, float)):
+        f = float(value)
+        return None if math.isnan(f) or math.isinf(f) else f
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, np.ndarray):
+        return [_json_safe(v) for v in value.tolist()]
+    if isinstance(value, (list, tuple)):
+        return [_json_safe(v) for v in value]
+    if isinstance(value, Mapping):
+        return {str(k): _json_safe(v) for k, v in value.items()}
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    return value
+
+
+def _fmt(value: Any, digits: int) -> str:
+    if value is None:
+        return "–"
+    if isinstance(value, (float, np.floating)):
+        return "NaN" if math.isnan(value) else f"{float(value):.{digits}f}"
+    return str(value)
+
+
+def _latex_escape(text: str) -> str:
+    repl = {
+        "\\": r"\textbackslash{}",
+        "&": r"\&",
+        "%": r"\%",
+        "$": r"\$",
+        "#": r"\#",
+        "_": r"\_",
+        "{": r"\{",
+        "}": r"\}",
+        "~": r"\textasciitilde{}",
+        "^": r"\textasciicircum{}",
+    }
+    return "".join(repl.get(c, c) for c in text)
+
+
+@dataclass(frozen=True, eq=False)
+class MetricResult:
+    """One metric's value.
+
+    ``value`` is a float, or a read-only array of per-class values when ``average=None``
+    (aligned with ``labels``). ``params`` records how it was computed (average, zero_division, ...).
+    """
+
+    metric: str
+    name: str
+    value: Union[float, np.ndarray]
+    params: Mapping[str, Any] = field(default_factory=dict)
+    labels: Optional[tuple[Any, ...]] = None
+
+    def __post_init__(self) -> None:
+        if isinstance(self.value, np.ndarray):
+            arr = np.array(self.value, dtype=np.float64)
+            arr.setflags(write=False)
+            object.__setattr__(self, "value", arr)
+        else:
+            object.__setattr__(self, "value", float(self.value))
+        object.__setattr__(self, "params", MappingProxyType(dict(self.params)))
+
+    def __float__(self) -> float:
+        if isinstance(self.value, np.ndarray):
+            raise TypeError(f"{self.name} has per-class values; use .value, .per_class() or .to_dict().")
+        return float(self.value)
+
+    # Scalar results behave like numbers: f"{r:.3f}", r > 0.8, round(r, 3), r == 0.75.
+    def __format__(self, spec: str) -> str:
+        return format(float(self), spec) if spec else repr(self)
+
+    def __round__(self, ndigits: Optional[int] = None) -> float:
+        return round(float(self), ndigits)
+
+    def __lt__(self, other: Number) -> bool:
+        return float(self) < float(other)
+
+    def __le__(self, other: Number) -> bool:
+        return float(self) <= float(other)
+
+    def __gt__(self, other: Number) -> bool:
+        return float(self) > float(other)
+
+    def __ge__(self, other: Number) -> bool:
+        return float(self) >= float(other)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, (int, float, np.floating, np.integer)) and not isinstance(self.value, np.ndarray):
+            return float(self.value) == float(other)
+        return self is other
+
+    def __hash__(self) -> int:
+        return id(self)
+
+    def __repr__(self) -> str:
+        if isinstance(self.value, np.ndarray):
+            pairs = ", ".join(f"{lab}: {_fmt(v, 4)}" for lab, v in zip(self.labels or (), self.value))
+            return f"MetricResult({self.metric}: {{{pairs}}})"
+        return f"MetricResult({self.metric}={_fmt(self.value, 6)})"
+
+    def per_class(self) -> dict[Any, float]:
+        if not isinstance(self.value, np.ndarray):
+            raise TypeError(f"{self.name} is a single value; compute it with average=None for per-class values.")
+        return {lab: float(v) for lab, v in zip(self.labels or (), self.value)}
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {"metric": self.metric, "name": self.name, "value": _json_safe(self.value)}
+        if self.labels is not None:
+            out["labels"] = _json_safe(list(self.labels))
+        if self.params:
+            out["params"] = _json_safe(dict(self.params))
+        return out
+
+    def to_json(self, *, indent: Optional[int] = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent, allow_nan=False)
+
+    def to_dataframe(self) -> pd.DataFrame:
+        import pandas as pd
+
+        frame: pd.DataFrame
+        if isinstance(self.value, np.ndarray):
+            frame = pd.DataFrame({"label": list(self.labels or ()), self.metric: self.value})
+        else:
+            frame = pd.DataFrame({"metric": [self.metric], "name": [self.name], "value": [self.value]})
+        return frame
+
+    def to_markdown(self, *, digits: int = 4) -> str:
+        if isinstance(self.value, np.ndarray):
+            rows = [f"| {lab} | {_fmt(v, digits)} |" for lab, v in zip(self.labels or (), self.value)]
+            return "\n".join([f"| Label | {self.name} |", "| --- | ---: |", *rows])
+        return "\n".join(["| Metric | Value |", "| --- | ---: |", f"| {self.name} | {_fmt(self.value, digits)} |"])
+
+    def to_latex(self, *, digits: int = 4) -> str:
+        return _latex_table(
+            ["Metric", "Value"] if not isinstance(self.value, np.ndarray) else ["Label", _latex_escape(self.name)],
+            [[_latex_escape(self.name), _fmt(self.value, digits)]]
+            if not isinstance(self.value, np.ndarray)
+            else [[_latex_escape(str(lab)), _fmt(v, digits)] for lab, v in zip(self.labels or (), self.value)],
+        )
+
+
+def _latex_table(
+    header: list[str], rows: list[list[str]], caption: Optional[str] = None, label: Optional[str] = None
+) -> str:
+    cols = "l" + "r" * (len(header) - 1)
+    lines = [r"\begin{table}[ht]", r"\centering"]
+    if caption:
+        lines.append(rf"\caption{{{_latex_escape(caption)}}}")
+    if label:
+        lines.append(rf"\label{{{label}}}")
+    lines += [rf"\begin{{tabular}}{{{cols}}}", r"\toprule", " & ".join(header) + r" \\", r"\midrule"]
+    lines += [" & ".join(r) + r" \\" for r in rows]
+    lines += [r"\bottomrule", r"\end{tabular}", r"\end{table}"]
+    return "\n".join(lines)
+
+
+@dataclass(frozen=True, eq=False)
+class EvaluationResult(Mapping[str, MetricResult]):
+    """All metrics from one evaluation, keyed by metric id (e.g. ``"accuracy"``)."""
+
+    task: str
+    metrics: Mapping[str, MetricResult]
+    n_samples: int
+    target_type: str
+    labels: Optional[tuple[Any, ...]] = None
+    confusion_matrix: Optional[np.ndarray] = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "metrics", MappingProxyType(dict(self.metrics)))
+        object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata)))
+        if self.confusion_matrix is not None:
+            cm = np.array(self.confusion_matrix)
+            cm.setflags(write=False)
+            object.__setattr__(self, "confusion_matrix", cm)
+
+    # Mapping interface: result["f1"]
+    def __getitem__(self, key: str) -> MetricResult:
+        try:
+            return self.metrics[key]
+        except KeyError:
+            raise KeyError(f"No metric '{key}' in this result. Available: {', '.join(self.metrics)}") from None
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.metrics)
+
+    def __len__(self) -> int:
+        return len(self.metrics)
+
+    @property
+    def value(self) -> dict[str, Any]:
+        return {k: m.value for k, m in self.metrics.items()}
+
+    def summary(self, *, digits: int = 4) -> str:
+        width = max((len(m.name) for m in self.metrics.values()), default=6)
+        lines = [f"EvalSuite {self.task} evaluation ({self.target_type}, n={self.n_samples})"]
+        for m in self.metrics.values():
+            if isinstance(m.value, np.ndarray):
+                lines.append(
+                    f"  {m.name:<{width}}  "
+                    + ", ".join(f"{lab}={_fmt(v, digits)}" for lab, v in zip(m.labels or (), m.value))
+                )
+            else:
+                lines.append(f"  {m.name:<{width}}  {_fmt(m.value, digits)}")
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:
+        return self.summary()
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "task": self.task,
+            "target_type": self.target_type,
+            "n_samples": self.n_samples,
+            "metrics": {k: m.to_dict() for k, m in self.metrics.items()},
+            "metadata": _json_safe(dict(self.metadata)),
+        }
+        if self.labels is not None:
+            out["labels"] = _json_safe(list(self.labels))
+        if self.confusion_matrix is not None:
+            out["confusion_matrix"] = _json_safe(self.confusion_matrix)
+        return out
+
+    def to_json(self, *, indent: Optional[int] = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent, allow_nan=False)
+
+    def _scalar_rows(self) -> list[MetricResult]:
+        return [m for m in self.metrics.values() if not isinstance(m.value, np.ndarray)]
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """One row per scalar metric (per-class metrics: use ``result[id].to_dataframe()``)."""
+        import pandas as pd
+
+        rows = self._scalar_rows()
+        frame: pd.DataFrame = pd.DataFrame(
+            {
+                "metric": [m.metric for m in rows],
+                "name": [m.name for m in rows],
+                "value": [m.value for m in rows],
+            }
+        )
+        return frame
+
+    def to_markdown(self, *, digits: int = 4) -> str:
+        rows = [f"| {m.name} | {_fmt(m.value, digits)} |" for m in self._scalar_rows()]
+        return "\n".join(["| Metric | Value |", "| --- | ---: |", *rows])
+
+    def to_latex(self, *, digits: int = 4, caption: Optional[str] = None, label: Optional[str] = None) -> str:
+        rows = [[_latex_escape(m.name), _fmt(m.value, digits)] for m in self._scalar_rows()]
+        return _latex_table(["Metric", "Value"], rows, caption=caption, label=label)

evalsuite/core/types.py

@@ -0,0 +1,23 @@
+"""Shared type aliases."""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Union
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+__all__ = ["Average", "ArrayLike", "FloatArray", "IntArray", "Task", "ZeroDivision"]
+
+FloatArray = NDArray[np.float64]
+IntArray = NDArray[np.int64]
+
+#: How per-class results are combined. ``None`` returns per-class values.
+Average = Literal["binary", "micro", "macro", "weighted", "samples", None]
+
+#: Value used when a metric's denominator is zero. ``"warn"`` returns NaN and warns.
+ZeroDivision = Union[Literal["warn"], float]
+
+Task = Literal["classification", "regression"]
+
+JSONValue = Union[None, bool, int, float, str, list[Any], dict[str, Any]]