assay-engine 0.5.0.dev2__py3-none-any.whl

assay/metrics.py

@@ -0,0 +1,238 @@
+"""Deterministic classification metrics — thin wrappers over scikit-learn.
+
+Assay reimplements no metric math; it validates inputs, delegates to sklearn, and
+returns an immutable, fully-typed result."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol, SupportsFloat, SupportsInt, cast
+
+from assay._optional import call_dependency, dependency_failed, load_callable, load_module
+from assay.errors import InvalidScoreRequest
+from assay.limits import MAX_ITEMS
+
+_MIN_CLASSES = 2
+_PRF_RESULT_COUNT = 3
+_CONFUSION_CELL_COUNT = 4
+_METRICS_MODULES = ("numpy", "scipy", "sklearn", "ir_measures", "pydantic_settings")
+
+_BINARY_LABELS = (0, 1)
+"""The confusion matrix is pinned to these, in this order, so its orientation can never
+depend on which labels happen to appear in the data."""
+
+
+def require_metrics_extra() -> None:
+    """Refuse optional calculators with one stable, value-free error."""
+    for module in _METRICS_MODULES:
+        load_module(module)
+
+
+@dataclass(frozen=True)
+class ConfusionCounts:
+    """The four cells of a binary confusion matrix at one threshold.
+
+    Named cells, not a bare 2x2 array. ``confusion_matrix(...).ravel()`` returns them in
+    the order ``tn, fp, fn, tp``, and reading that tuple in the wrong order is the
+    classic silent inversion: it swaps a miss for a false alarm while every total still
+    adds up, so nothing downstream can notice."""
+
+    true_positives: int
+    false_positives: int
+    true_negatives: int
+    false_negatives: int
+
+
+@dataclass(frozen=True)
+class ClassificationScores:
+    """Immutable bundle of binary-classification metrics."""
+
+    accuracy: float
+    precision: float
+    recall: float
+    f1: float
+    pr_auc: float
+    roc_auc: float
+    counts: ConfusionCounts
+    false_negative_rate: float
+
+
+class _RavelResult(Protocol):
+    def ravel(self) -> Sequence[object]: ...
+
+
+def _validate_shape(y_true: Sequence[int], y_score: Sequence[float]) -> None:
+    if len(y_true) != len(y_score) or len(y_true) > MAX_ITEMS:
+        raise InvalidScoreRequest
+    if not y_true:
+        raise InvalidScoreRequest
+
+
+def _validate(y_true: Sequence[int], y_score: Sequence[float]) -> None:
+    _validate_shape(y_true, y_score)
+    if not all(math.isfinite(score) for score in y_score):
+        raise InvalidScoreRequest
+    _require_binary_labels(y_true)
+
+
+def _require_auc_classes(y_true: Sequence[int]) -> None:
+    if len(set(y_true)) < _MIN_CLASSES:
+        raise InvalidScoreRequest
+
+
+def _threshold(y_score: Sequence[float], threshold: float) -> list[int]:
+    if isinstance(threshold, bool) or not math.isfinite(threshold):
+        raise InvalidScoreRequest
+    return [1 if s >= threshold else 0 for s in y_score]
+
+
+def _prf(y_true: Sequence[int], y_pred: Sequence[int]) -> tuple[float, float, float]:
+    raw = _metric_call(
+        "precision_recall_fscore_support",
+        y_true,
+        y_pred,
+        average="binary",
+        zero_division=0.0,
+    )
+    values = call_dependency(_first_three_finite, raw)
+    if dependency_failed(values):
+        raise InvalidScoreRequest
+    return cast(tuple[float, float, float], values)
+
+
+def _first_three_finite(raw: object) -> tuple[float, float, float]:
+    values = cast(Sequence[object], raw)
+    result = tuple(_as_float(value) for value in values[:_PRF_RESULT_COUNT])
+    if len(result) != _PRF_RESULT_COUNT or not all(math.isfinite(value) for value in result):
+        raise ValueError
+    first, second, third = result
+    return first, second, third
+
+
+def _as_float(value: object) -> float:
+    return float(cast(SupportsFloat, value))
+
+
+def _as_int(value: object) -> int:
+    return int(cast(SupportsInt, value))
+
+
+def _require_binary_labels(y_true: Sequence[int]) -> None:
+    """Every label must be 0 or 1.
+
+    This refusal is what makes pinning ``labels=(0, 1)`` safe. sklearn silently DROPS
+    every row whose label falls outside that pinning, so a stray 2 would vanish from the
+    counts and the four cells would come back looking perfectly healthy, computed over
+    fewer examples than the caller handed in."""
+    outside = set(y_true) - set(_BINARY_LABELS)
+    if outside:
+        raise InvalidScoreRequest
+
+
+def _counts(y_true: Sequence[int], y_pred: Sequence[int]) -> ConfusionCounts:
+    raw = _metric_call("confusion_matrix", y_true, y_pred, labels=_BINARY_LABELS)
+    cells = call_dependency(_confusion_cells, raw)
+    if dependency_failed(cells):
+        raise InvalidScoreRequest
+    true_negatives, false_positives, false_negatives, true_positives = cast(
+        tuple[int, int, int, int], cells
+    )
+    return ConfusionCounts(
+        true_positives=int(true_positives),
+        false_positives=int(false_positives),
+        true_negatives=int(true_negatives),
+        false_negatives=int(false_negatives),
+    )
+
+
+def _confusion_cells(raw: object) -> tuple[int, int, int, int]:
+    values = cast(_RavelResult, raw).ravel()
+    cells = tuple(_as_int(value) for value in values)
+    if len(cells) != _CONFUSION_CELL_COUNT or any(value < 0 for value in cells):
+        raise ValueError
+    return cast(tuple[int, int, int, int], cells)
+
+
+def confusion_counts(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> ConfusionCounts:
+    """The four confusion cells at ``threshold``, from sklearn's confusion matrix.
+
+    A rate hides which way a system fails. 200 misses and 2 false alarms produce the same
+    accuracy as 2 misses and 200 false alarms, and only the counts tell them apart."""
+    _validate(y_true, y_score)
+    _require_binary_labels(y_true)
+    return _counts(y_true, _threshold(y_score, threshold))
+
+
+def _false_negative_rate(counts: ConfusionCounts) -> float:
+    """Misses over real positives. The denominator cannot be zero: ``_validate`` already
+    requires both classes in ``y_true``, so at least one real positive exists."""
+    positives = counts.false_negatives + counts.true_positives
+    if positives == 0:
+        raise InvalidScoreRequest
+    return counts.false_negatives / positives
+
+
+def false_negative_rate(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> float:
+    """The miss rate: of everything that really was positive, what fraction was called
+    negative.
+
+    Exactly ``1 - recall``, and named anyway. For a screening system the miss is the
+    number it is judged on, and nobody reads a 3% miss rate off a recall of 0.97."""
+    return _false_negative_rate(confusion_counts(y_true, y_score, threshold=threshold))
+
+
+def _scores(
+    y_true: Sequence[int], y_score: Sequence[float], y_pred: Sequence[int]
+) -> ClassificationScores:
+    precision, recall, f1 = _prf(y_true, y_pred)
+    counts = _counts(y_true, y_pred)
+    return ClassificationScores(
+        accuracy=_metric_float("accuracy_score", y_true, y_pred),
+        precision=precision,
+        recall=recall,
+        f1=f1,
+        pr_auc=_metric_float("average_precision_score", y_true, y_score),
+        roc_auc=_metric_float("roc_auc_score", y_true, y_score),
+        counts=counts,
+        false_negative_rate=_false_negative_rate(counts),
+    )
+
+
+def _metric_call(name: str, *args: object, **kwargs: object) -> object:
+    result = call_dependency(load_callable("sklearn.metrics", name), *args, **kwargs)
+    if dependency_failed(result):
+        raise InvalidScoreRequest
+    return result
+
+
+def _metric_float(name: str, *args: object) -> float:
+    raw = _metric_call(name, *args)
+    result = call_dependency(float, raw)
+    if dependency_failed(result) or not math.isfinite(cast(float, result)):
+        raise InvalidScoreRequest
+    return cast(float, result)
+
+
+def binary_scores(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> ClassificationScores:
+    """Compute accuracy, precision, recall, F1, PR-AUC, ROC-AUC, the four confusion
+    counts and the false-negative rate."""
+    _validate(y_true, y_score)
+    _require_auc_classes(y_true)
+    return _scores(y_true, y_score, _threshold(y_score, threshold))
+
+
+def correctness(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> tuple[float, ...]:
+    """Return a per-example 1.0/0.0 correctness vector (for bootstrapping)."""
+    _validate(y_true, y_score)
+    y_pred = _threshold(y_score, threshold)
+    return tuple(float(int(p == t)) for p, t in zip(y_pred, y_true, strict=True))

assay/minimum.py

@@ -0,0 +1,82 @@
+"""First-occurrence minimum composition over normalized components."""
+
+from __future__ import annotations
+
+from assay.composite import inputs_hash, interval_or_none
+from assay.contracts import (
+    Component,
+    ExplainedComponent,
+    Interval,
+    Method,
+    MinimumRequest,
+    Operation,
+    ScoreResult,
+)
+from assay.normalize import normalize
+
+
+def _normalized(component: Component, request: MinimumRequest, value: float) -> float:
+    return normalize(value, component.scale, request.clamp)
+
+
+def _explain(component: Component, request: MinimumRequest) -> ExplainedComponent:
+    candidate = _normalized(component, request, component.value)
+    return ExplainedComponent(
+        id=component.id,
+        raw=component.value,
+        normalized=candidate,
+        declared_weight=None,
+        operation=Operation.ADD,
+        coefficient=1.0,
+        contribution=candidate,
+        contribution_interval=_candidate_interval(component, request),
+    )
+
+
+def _bounds(component: Component, request: MinimumRequest) -> tuple[float, float]:
+    interval = component.interval
+    if interval is None:
+        point = _normalized(component, request, component.value)
+        return point, point
+    first = _normalized(component, request, interval.low)
+    second = _normalized(component, request, interval.high)
+    return min(first, second), max(first, second)
+
+
+def _candidate_interval(component: Component, request: MinimumRequest) -> Interval | None:
+    if component.interval is None:
+        return None
+    return interval_or_none(*_bounds(component, request))
+
+
+def _propagated_bounds(request: MinimumRequest) -> tuple[float, float]:
+    bounds = tuple(_bounds(component, request) for component in request.components)
+    return min(low for low, _ in bounds), min(high for _, high in bounds)
+
+
+def _result_interval(request: MinimumRequest) -> Interval | None:
+    if not any(component.interval is not None for component in request.components):
+        return None
+    return interval_or_none(*_propagated_bounds(request))
+
+
+def _result(request: MinimumRequest, rows: tuple[ExplainedComponent, ...]) -> ScoreResult:
+    selected = min(rows, key=lambda row: row.contribution)
+    return ScoreResult(
+        method=Method(id=request.method, version=request.method_version),
+        score=selected.contribution,
+        interval=_result_interval(request),
+        clamp=request.clamp,
+        intercept=None,
+        weight_total=None,
+        components=rows,
+        inputs_hash=inputs_hash(request),
+        selected_component_id=selected.id,
+    )
+
+
+def minimum(request: MinimumRequest) -> ScoreResult:
+    """Return the first lowest normalized candidate and identify it explicitly."""
+    validated = MinimumRequest.model_validate(request)
+    rows = tuple(_explain(component, validated) for component in validated.components)
+    return _result(validated, rows)

assay/models.py

@@ -0,0 +1,111 @@
+"""Typed request models at the input boundary. Frozen and ``extra="forbid"`` so a
+malformed or ambiguous request is rejected before any computation."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ScoreRequest(BaseModel):
+    """A classification scoring request."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str
+    metric_version: str
+    y_true: tuple[int, ...]
+    y_score: tuple[float, ...]
+    threshold: float = 0.5
+
+
+class SubScoreInput(BaseModel):
+    """One sub-score with its native scale and interval, for a composite."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    name: str
+    value: float
+    low: float
+    high: float
+    scale_min: float
+    scale_max: float
+    weight: float
+
+
+class CompositeRequest(BaseModel):
+    """A weighted multi-scale composite request."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str = "weighted_composite"
+    metric_version: str
+    subscores: tuple[SubScoreInput, ...]
+
+
+class RelevanceJudgment(BaseModel):
+    """How relevant one document is to one query.
+
+    ``gain`` 0 means judged and not relevant; any positive gain means relevant, and a
+    larger gain means more relevant. Binary judgments are the special case where every
+    listed document has gain 1.0."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    doc_id: str
+    gain: float = 1.0
+
+
+class RankedQuery(BaseModel):
+    """One query's scoring inputs: the returned order and relevance judgments.
+
+    ``ranked`` is a *position* list — first element is the top hit — not scores. The two
+    are deliberately separate: a ranking is judged against the whole judgment set,
+    including relevant documents the system never returned."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    query: str
+    judgments: tuple[RelevanceJudgment, ...]
+    ranked: tuple[str, ...]
+
+
+class ItemRating(BaseModel):
+    """One item, and the band each of two raters put it in.
+
+    Item-keyed rather than two loose parallel lists: the id is what makes "the same item
+    graded twice" detectable, and a silent duplicate lets one disputed item vote twice."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    item: str
+    rater_a: str
+    rater_b: str
+
+
+class AgreementRequest(BaseModel):
+    """An inter-rater agreement request over one set of doubly-graded items.
+
+    ``scale`` is ORDERED, weakest band first. The declared order is part of the scoring
+    method and result explanation because changing it changes the measurement."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str = "agreement"
+    metric_version: str
+    scale: tuple[str, ...]
+    ratings: tuple[ItemRating, ...]
+
+
+class RankingRequest(BaseModel):
+    """A ranked-retrieval scoring request over a whole query set.
+
+    ``k`` left as ``None`` means "use ``AssaySettings.ranking_k``"; whichever value ends
+    up applying is method provenance, so a reported precision@k always explains which k
+    it used."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str = "ranking"
+    metric_version: str
+    queries: tuple[RankedQuery, ...]
+    k: int | None = None

assay/normalize.py

@@ -0,0 +1,74 @@
+"""Pure normalization from a declared native scale to zero through one."""
+
+from __future__ import annotations
+
+import math
+from typing import Final, NoReturn
+
+from assay.contracts import ClampPolicy, Direction, NativeScale
+from assay.errors import ContractCode, ContractValidationError
+
+__all__ = ["normalize"]
+
+_NORMALIZED_MIN: Final = 0.0
+_NORMALIZED_MAX: Final = 1.0
+
+
+def _fail(code: ContractCode) -> NoReturn:
+    raise ContractValidationError(code) from None
+
+
+def _finite(value: object) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        _fail(ContractCode.INVALID_NUMBER)
+    try:
+        number = float(value)
+    except OverflowError:
+        _fail(ContractCode.INVALID_NUMBER)
+    if not math.isfinite(number):
+        _fail(ContractCode.INVALID_NUMBER)
+    return number
+
+
+def _formula(value: float, scale: NativeScale) -> float:
+    width = scale.maximum - scale.minimum
+    offset = value - scale.minimum
+    if scale.direction is Direction.LOWER_IS_BETTER:
+        offset = scale.maximum - value
+    if not math.isfinite(width) or not math.isfinite(offset):
+        _fail(ContractCode.INVALID_NUMBER)
+    result = offset / width
+    if not math.isfinite(result):
+        _fail(ContractCode.INVALID_NUMBER)
+    return result
+
+
+def _canonical_policy(clamp: ClampPolicy) -> ClampPolicy:
+    if not isinstance(clamp, ClampPolicy):
+        _fail(ContractCode.INVALID_CLAMP_POLICY)
+    try:
+        return ClampPolicy(str(clamp))
+    except ValueError:
+        _fail(ContractCode.INVALID_CLAMP_POLICY)
+
+
+def _canonical_zero(value: float) -> float:
+    return _NORMALIZED_MIN if value == _NORMALIZED_MIN else value
+
+
+def _apply_policy(result: float, clamp: ClampPolicy) -> float:
+    policy = _canonical_policy(clamp)
+    if policy is ClampPolicy.CLAMP:
+        bounded = min(_NORMALIZED_MAX, max(_NORMALIZED_MIN, result))
+        return _canonical_zero(bounded)
+    if not _NORMALIZED_MIN <= result <= _NORMALIZED_MAX:
+        _fail(ContractCode.OUT_OF_RANGE)
+    return _canonical_zero(result)
+
+
+def normalize(value: float, scale: NativeScale, clamp: ClampPolicy) -> float:
+    """Map one finite native value onto its declared zero-to-one scale."""
+    number = _finite(value)
+    validated_scale = NativeScale.model_validate(scale)
+    result = _formula(number, validated_scale)
+    return _apply_policy(result, clamp)