assay-engine 0.5.0.dev2__py3-none-any.whl

assay/agreement.py

@@ -0,0 +1,312 @@
+"""Ordinal agreement — did two graders mean the same thing, or just share a habit?
+
+``metrics.py`` scores predictions against ground truth. This module scores two *raters*
+against each other, when what they emit is a band on an ordered scale (weak / moderate /
+strong) rather than a number. That shape has no ground truth to score against: nobody
+knows the true band, only whether two independent graders landed in the same place.
+
+**Why percent agreement is the wrong statistic here**, and the reason this module exists:
+
+1. It is blind to *distance*. On a three-band scale, one grader saying "strong" while the
+   other says "moderate" is a near miss; "strong" against "weak" is a total miss. Percent
+   agreement scores both as simply "not a match".
+2. It counts agreement that chance alone would produce. Two graders who both call 90% of
+   everything "weak" will match about 80% of the time while agreeing about nothing.
+
+One-line definitions, since none of these terms carry themselves:
+
+- **quadratic-weighted Cohen's kappa** — agreement after subtracting the agreement two
+  independent graders with these same habits would have produced, with each miss charged
+  by the *square* of how many bands apart it was. 1.0 is perfect, 0.0 is exactly chance,
+  and negative means the two graders did worse than if they had ignored each other.
+- **Kendall's tau-b** — of every pair of items, how often the two graders put them in the
+  same relative order. +1 perfectly concordant, -1 perfectly inverted. The ``b`` is the
+  tie correction, which a three-band scale over many items needs badly.
+
+They answer different questions and the report carries both: two graders who agree on
+every *ordering* but sit one band apart on the level score tau-b 1.0 and kappa 2/3.
+
+*The engines.* Kappa is ``sklearn.metrics.cohen_kappa_score(weights="quadratic")`` and
+tau-b is ``scipy.stats.kendalltau(variant="b")`` — both already pinned dependencies, both
+the reference implementation their field validates against, and neither needs a line of
+correction at the boundary. Assay contributes what they do not: a declared band order
+that cannot be guessed wrong, a refusal for every input whose answer would be undefined,
+and an uncertainty interval on the same honesty floor the rest of the package uses.
+
+*The trap this module is built around.* ``cohen_kappa_score`` derives the ordinal
+distance between two bands from their **positions in its ``labels`` argument**. Leave
+that argument off and it sorts the band names alphabetically, so "moderate < strong <
+weak" — and the same ratings come back with a completely different, entirely plausible
+number. Sorted-by-accident is not a scale. The caller declares the order, always.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, cast
+
+from pydantic import BaseModel, ConfigDict
+
+from assay._optional import call_dependency, dependency_failed, load_callable, load_module
+from assay.errors import InvalidAgreementRequest
+from assay.limits import MAX_ITEMS, MAX_SCALE_LEVELS
+from assay.models import ItemRating
+from assay.uncertainty import Estimate, mean_interval
+
+if TYPE_CHECKING:
+    from assay.settings import AssaySettings
+
+type Scale = Sequence[str]
+"""The band names in order, weakest first. The order IS the measurement."""
+
+_MIN_LEVELS = 2
+"""Below two bands there is nothing to disagree about, and the quadratic weight's
+``L - 1`` denominator is zero."""
+
+_KAPPA_UNDEFINED = (
+    "both raters put every item in the same single band, so agreement by chance is total "
+    "and kappa's correction divides by zero"
+)
+_TAU_UNDEFINED = (
+    "at least one rater used a single band for every item, so there is no rank variation "
+    "for tau-b to concord"
+)
+
+
+class _StatisticResult(Protocol):
+    statistic: object
+
+
+__all__ = [
+    "AgreementReport",
+    "Scale",
+    "agreement_report",
+    "kendall_tau_b",
+    "percent_agreement",
+    "quadratic_kappa",
+    "weighted_agreement",
+]
+
+
+class AgreementReport(BaseModel):
+    """Two graders on one ordinal scale, and how much of their agreement is real.
+
+    ``weighted_agreement_interval`` is a bootstrap confidence interval over the per-item
+    quadratic agreement, or an ``Abstention`` when there are fewer items than the sample
+    floor — the same uncertainty story the classification and ranking faces tell, not a
+    third one. It is deliberately NOT an interval on kappa: kappa is not the mean of any
+    per-item quantity, so no bootstrap of a mean can put an interval on it, and pairing
+    it with one would be exactly the fake precision this package refuses to print.
+
+    ``quadratic_kappa`` and ``kendall_tau_b`` are ``None`` when the ratings are too
+    degenerate for the statistic to exist, and the matching ``*_undefined_reason`` says
+    which degeneracy it was. A bare ``None`` reads as "not computed"; the reason says
+    "cannot exist", which is a different fact about the data."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    scale: tuple[str, ...]
+    n_items: int
+    n_exact_matches: int
+    percent_agreement: float
+    weighted_agreement: float
+    quadratic_kappa: float | None
+    kappa_undefined_reason: str | None
+    kendall_tau_b: float | None
+    tau_undefined_reason: str | None
+    weighted_agreement_interval: Estimate
+
+
+def _require_scale(scale: Scale) -> None:
+    if not _MIN_LEVELS <= len(scale) <= MAX_SCALE_LEVELS:
+        raise InvalidAgreementRequest(
+            f"an ordinal scale needs at least {_MIN_LEVELS} bands, got {len(scale)}"
+        )
+    if len(set(scale)) != len(scale):
+        raise InvalidAgreementRequest("the scale names the same band twice")
+
+
+def _require_paired(rater_a: Sequence[str], rater_b: Sequence[str]) -> None:
+    if len(rater_a) != len(rater_b):
+        raise InvalidAgreementRequest(
+            f"the raters graded different numbers of items: {len(rater_a)} vs {len(rater_b)}"
+        )
+    if not rater_a:
+        raise InvalidAgreementRequest("no items were graded; there is nothing to agree about")
+    if len(rater_a) > MAX_ITEMS:
+        raise InvalidAgreementRequest
+
+
+def _ordinals(ratings: Sequence[str], positions: Mapping[str, int]) -> list[int]:
+    """Band names as positions on the declared scale, refusing any band not on it.
+
+    The refusal is load-bearing, not defensive. ``cohen_kappa_score`` silently DROPS
+    every row whose label is outside its ``labels`` argument, so an unknown band would
+    return a perfectly healthy number computed over fewer items than were handed in."""
+    unknown = sorted(set(ratings) - set(positions))
+    if unknown:
+        raise InvalidAgreementRequest(f"bands that are not on the declared scale: {unknown}")
+    return [positions[band] for band in ratings]
+
+
+def _validate(
+    rater_a: Sequence[str], rater_b: Sequence[str], scale: Scale
+) -> tuple[list[int], list[int]]:
+    """Check everything, then return both raters as positions on the declared scale."""
+    _require_scale(scale)
+    _require_paired(rater_a, rater_b)
+    positions = {band: index for index, band in enumerate(scale)}
+    return _ordinals(rater_a, positions), _ordinals(rater_b, positions)
+
+
+def _per_item_weights(rater_a: Sequence[int], rater_b: Sequence[int], levels: int) -> list[float]:
+    """Cohen's quadratic weight per item: ``1 - ((i - j) / (L - 1)) ** 2``.
+
+    1.0 for an exact match, falling with the SQUARE of the distance between bands — on a
+    three-band scale an adjacent miss keeps 0.75 and an opposite-ends miss keeps 0.0."""
+    span = levels - 1
+    return [1.0 - ((a - b) / span) ** 2 for a, b in zip(rater_a, rater_b, strict=True)]
+
+
+def percent_agreement(rater_a: Sequence[str], rater_b: Sequence[str], *, scale: Scale) -> float:
+    """The fraction of items both raters put in exactly the same band.
+
+    Carried because it is the number people reach for — and because the report exists to
+    show, side by side, why it is not enough."""
+    load_module("numpy")
+    ordinals_a, ordinals_b = _validate(rater_a, rater_b, scale)
+    matches = sum(a == b for a, b in zip(ordinals_a, ordinals_b, strict=True))
+    return matches / len(ordinals_a)
+
+
+def weighted_agreement(rater_a: Sequence[str], rater_b: Sequence[str], *, scale: Scale) -> float:
+    """Mean per-item quadratic agreement: like percent agreement, but a near miss counts.
+
+    Still uncorrected for chance — that correction is what kappa adds on top of it."""
+    ordinals_a, ordinals_b = _validate(rater_a, rater_b, scale)
+    return _numpy_mean(_per_item_weights(ordinals_a, ordinals_b, len(scale)))
+
+
+def quadratic_kappa(
+    rater_a: Sequence[str], rater_b: Sequence[str], *, scale: Scale
+) -> float | None:
+    """Sklearn quadratic kappa, or ``None`` when chance agreement is total."""
+    _validate(rater_a, rater_b, scale)
+    if len(set(rater_a) | set(rater_b)) < _MIN_LEVELS:
+        return None
+    return _kappa_score(rater_a, rater_b, scale)
+
+
+def _kappa_score(rater_a: Sequence[str], rater_b: Sequence[str], scale: Scale) -> float:
+    return _finite_float(
+        _call(
+            "sklearn.metrics",
+            "cohen_kappa_score",
+            rater_a,
+            rater_b,
+            labels=list(scale),
+            weights="quadratic",
+        )
+    )
+
+
+def kendall_tau_b(rater_a: Sequence[str], rater_b: Sequence[str], *, scale: Scale) -> float | None:
+    """Tie-corrected rank concordance, or ``None`` for a constant rater."""
+    ordinals_a, ordinals_b = _validate(rater_a, rater_b, scale)
+    if min(len(set(ordinals_a)), len(set(ordinals_b))) < _MIN_LEVELS:
+        return None
+    raw = _call("scipy.stats", "kendalltau", ordinals_a, ordinals_b, variant="b")
+    statistic = call_dependency(_statistic, raw)
+    if dependency_failed(statistic):
+        raise InvalidAgreementRequest
+    return _finite_float(statistic)
+
+
+def _statistic(value: object) -> object:
+    return cast(_StatisticResult, value).statistic
+
+
+def _call(module: str, name: str, *args: object, **kwargs: object) -> object:
+    result = call_dependency(load_callable(module, name), *args, **kwargs)
+    if dependency_failed(result):
+        raise InvalidAgreementRequest
+    return result
+
+
+def _finite_float(value: object) -> float:
+    converted = call_dependency(float, value)
+    if dependency_failed(converted) or not math.isfinite(cast(float, converted)):
+        raise InvalidAgreementRequest
+    return cast(float, converted)
+
+
+def _numpy_mean(values: Sequence[float]) -> float:
+    return _finite_float(_call("numpy", "mean", values))
+
+
+def _require_distinct_items(ratings: Sequence[ItemRating]) -> None:
+    items = [row.item for row in ratings]
+    if len(set(items)) != len(items):
+        raise InvalidAgreementRequest("the same item is graded twice")
+
+
+def _interval(per_item: Sequence[float], settings: AssaySettings) -> Estimate:
+    return mean_interval(
+        per_item,
+        min_samples=settings.min_samples,
+        n_resamples=settings.bootstrap_resamples,
+        confidence_level=settings.confidence_level,
+        seed=settings.bootstrap_seed,
+    )
+
+
+@dataclass(frozen=True)
+class _ReportInputs:
+    rater_a: tuple[str, ...]
+    rater_b: tuple[str, ...]
+    scale: tuple[str, ...]
+    per_item: tuple[float, ...]
+
+
+def _statistics(data: _ReportInputs) -> tuple[float | None, float | None]:
+    kappa = quadratic_kappa(data.rater_a, data.rater_b, scale=data.scale)
+    tau = kendall_tau_b(data.rater_a, data.rater_b, scale=data.scale)
+    return kappa, tau
+
+
+def _report(data: _ReportInputs, settings: AssaySettings) -> AgreementReport:
+    kappa, tau = _statistics(data)
+    return AgreementReport(
+        scale=data.scale,
+        n_items=len(data.per_item),
+        n_exact_matches=sum(a == b for a, b in zip(data.rater_a, data.rater_b, strict=True)),
+        percent_agreement=percent_agreement(data.rater_a, data.rater_b, scale=data.scale),
+        weighted_agreement=_numpy_mean(data.per_item),
+        quadratic_kappa=kappa,
+        kappa_undefined_reason=None if kappa is not None else _KAPPA_UNDEFINED,
+        kendall_tau_b=tau,
+        tau_undefined_reason=None if tau is not None else _TAU_UNDEFINED,
+        weighted_agreement_interval=_interval(data.per_item, settings),
+    )
+
+
+def _report_inputs(ratings: Sequence[ItemRating], scale: Scale) -> _ReportInputs:
+    _require_distinct_items(ratings)
+    rater_a = [row.rater_a for row in ratings]
+    rater_b = [row.rater_b for row in ratings]
+    ordinals_a, ordinals_b = _validate(rater_a, rater_b, scale)
+    weights = _per_item_weights(ordinals_a, ordinals_b, len(scale))
+    return _ReportInputs(tuple(rater_a), tuple(rater_b), tuple(scale), tuple(weights))
+
+
+def agreement_report(
+    ratings: Sequence[ItemRating], *, scale: Scale, settings: AssaySettings
+) -> AgreementReport:
+    """Score one set of doubly-graded items: the counts, both statistics, and an interval.
+
+    ``ratings`` is item-keyed rather than two loose parallel lists, because the item id is
+    what makes "the same item graded twice" detectable — a duplicate would let one
+    disputed item vote twice and quietly reweight the whole measurement."""
+    return _report(_report_inputs(ratings, scale), settings)

assay/calibration.py

@@ -0,0 +1,167 @@
+"""Calibration evidence: Expected Calibration Error (ECE), a reliability diagram,
+and the Brier score.
+
+Reliability points come from ``sklearn.calibration.calibration_curve`` (which drops
+empty bins), bin populations from ``numpy.histogram`` over the same uniform edges,
+and Brier from ``sklearn.metrics.brier_score_loss``. ECE is the population-weighted
+gap between predicted confidence and observed frequency."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import SupportsInt, cast
+
+from assay._optional import call_dependency, dependency_failed, load_callable
+from assay.errors import InvalidScoreRequest
+from assay.limits import MAX_CALIBRATION_BINS, MAX_ITEMS
+
+
+@dataclass(frozen=True)
+class ReliabilityBin:
+    """One reliability-diagram point."""
+
+    mean_predicted: float
+    fraction_positive: float
+    count: int
+
+
+@dataclass(frozen=True)
+class CalibrationReport:
+    """ECE, Brier, and the reliability diagram for a set of predictions."""
+
+    ece: float
+    brier: float
+    bins: tuple[ReliabilityBin, ...]
+
+
+def _validate_shape(y_true: Sequence[int], y_score: Sequence[float], n_bins: int) -> None:
+    if len(y_true) != len(y_score) or not y_true:
+        raise InvalidScoreRequest
+    if len(y_true) > MAX_ITEMS:
+        raise InvalidScoreRequest
+    _validate_bin_count(n_bins)
+
+
+def _validate_bin_count(n_bins: int) -> None:
+    if isinstance(n_bins, bool) or not 0 < n_bins <= MAX_CALIBRATION_BINS:
+        raise InvalidScoreRequest
+
+
+def _validate_labels(y_true: Sequence[int]) -> None:
+    if set(y_true) - {0, 1}:
+        raise InvalidScoreRequest
+
+
+def _validate_scores(y_score: Sequence[float]) -> None:
+    if not all(math.isfinite(score) and 0.0 <= score <= 1.0 for score in y_score):
+        raise InvalidScoreRequest
+
+
+def _validate(y_true: Sequence[int], y_score: Sequence[float], n_bins: int) -> None:
+    _validate_shape(y_true, y_score, n_bins)
+    _validate_labels(y_true)
+    _validate_scores(y_score)
+
+
+def _bin_populations(score_arr: object, n_bins: int) -> list[int]:
+    # Bin with sklearn's own scheme (searchsorted on interior edges) rather than a
+    # parallel np.histogram: at an exact bin edge the two disagree, which would
+    # misalign these counts with calibration_curve's non-empty bins.
+    edges = _call("numpy", "linspace", 0.0, 1.0, n_bins + 1)
+    interior = call_dependency(_interior, edges)
+    if dependency_failed(interior):
+        raise InvalidScoreRequest
+    bin_ids = _call("numpy", "searchsorted", interior, score_arr)
+    counts = _call("numpy", "bincount", bin_ids, minlength=n_bins)
+    converted = call_dependency(_positive_ints, counts)
+    if dependency_failed(converted):
+        raise InvalidScoreRequest
+    return cast(list[int], converted)
+
+
+def _interior(values: object) -> object:
+    return values[1:-1]  # type: ignore[index]
+
+
+def _positive_ints(values: object) -> list[int]:
+    converted = [_as_int(value) for value in cast(Sequence[object], values)]
+    return [value for value in converted if value > 0]
+
+
+def _as_int(value: object) -> int:
+    return int(cast(SupportsInt, value))
+
+
+def _bins(
+    prob_pred: Sequence[float], prob_true: Sequence[float], weights: list[int]
+) -> list[ReliabilityBin]:
+    bins = [
+        ReliabilityBin(mean_predicted=float(p), fraction_positive=float(t), count=w)
+        for p, t, w in zip(prob_pred, prob_true, weights, strict=True)
+    ]
+    if not all(
+        math.isfinite(row.mean_predicted) and math.isfinite(row.fraction_positive) for row in bins
+    ):
+        raise InvalidScoreRequest
+    return bins
+
+
+def _ece(bins: list[ReliabilityBin], total: int) -> float:
+    result = sum(b.count / total * abs(b.mean_predicted - b.fraction_positive) for b in bins)
+    if not math.isfinite(result):
+        raise InvalidScoreRequest
+    return result
+
+
+def _call(module: str, name: str, *args: object, **kwargs: object) -> object:
+    result = call_dependency(load_callable(module, name), *args, **kwargs)
+    if dependency_failed(result):
+        raise InvalidScoreRequest
+    return result
+
+
+def _curve(
+    true_arr: object, score_arr: object, n_bins: int
+) -> tuple[Sequence[float], Sequence[float]]:
+    raw = _call(
+        "sklearn.calibration",
+        "calibration_curve",
+        true_arr,
+        score_arr,
+        n_bins=n_bins,
+        strategy="uniform",
+    )
+    converted = call_dependency(_pair, raw)
+    if dependency_failed(converted):
+        raise InvalidScoreRequest
+    return cast(tuple[Sequence[float], Sequence[float]], converted)
+
+
+def _pair(value: object) -> tuple[Sequence[float], Sequence[float]]:
+    first, second = cast(Sequence[Sequence[float]], value)
+    return first, second
+
+
+def _finite_float(value: object) -> float:
+    converted = call_dependency(float, value)
+    if dependency_failed(converted) or not math.isfinite(cast(float, converted)):
+        raise InvalidScoreRequest
+    return cast(float, converted)
+
+
+def calibration_report(
+    y_true: Sequence[int], y_score: Sequence[float], *, n_bins: int
+) -> CalibrationReport:
+    """Build the ECE / Brier / reliability report for binary predictions."""
+    _validate(y_true, y_score, n_bins)
+    true_arr = _call("numpy", "asarray", y_true, dtype=float)
+    score_arr = _call("numpy", "asarray", y_score, dtype=float)
+    prob_true, prob_pred = _curve(true_arr, score_arr, n_bins)
+    bins = _bins(prob_pred, prob_true, _bin_populations(score_arr, n_bins))
+    return CalibrationReport(
+        ece=_ece(bins, total=len(y_score)),
+        brier=_finite_float(_call("sklearn.metrics", "brier_score_loss", true_arr, score_arr)),
+        bins=tuple(bins),
+    )

assay/cli.py

@@ -0,0 +1,55 @@
+"""Dependency-light entry point with an early historical-command boundary."""
+
+from __future__ import annotations
+
+import sys
+from importlib import import_module
+from typing import Protocol, cast
+
+from assay.errors import AssayError, CliExtraMissing, CommandMovedToAvow, CommandReplaced
+
+_EXIT_USAGE = 2
+_MIGRATIONS = {
+    "keygen": f"FAIL: {CommandMovedToAvow.code}; use `avow keygen ...`\n",
+    "sign": f"FAIL: {CommandMovedToAvow.code}; use `avow sign ...`\n",
+    "verify": f"FAIL: {CommandMovedToAvow.code}; use `avow verify ...`\n",
+    "verify-ledger": f"FAIL: {CommandMovedToAvow.code}; use `avow ledger verify ...`\n",
+    "score": f"FAIL: {CommandReplaced.code}; use `assay measure ...`\n",
+    "composite": f"FAIL: {CommandReplaced.code}; use `assay compose ...`\n",
+}
+
+
+class _CommandRunner(Protocol):
+    def __call__(self, arguments: tuple[str, ...]) -> int: ...
+
+
+def _migration_message(arguments: tuple[str, ...]) -> str | None:
+    if not arguments:
+        return None
+    return _MIGRATIONS.get(arguments[0])
+
+
+def _run(arguments: tuple[str, ...]) -> int:
+    try:
+        module = import_module("assay._cli_app")
+    except ModuleNotFoundError:
+        raise CliExtraMissing from None
+    runner = cast(_CommandRunner, module.run)
+    return runner(arguments)
+
+
+def _fail(error: AssayError) -> int:
+    sys.stderr.write(f"FAIL: {error.code}\n")
+    return _EXIT_USAGE
+
+
+def main() -> int:
+    """Reject migrated commands before loading the scoring command adapter."""
+    message = _migration_message(tuple(sys.argv[1:]))
+    if message is not None:
+        sys.stderr.write(message)
+        return _EXIT_USAGE
+    try:
+        return _run(tuple(sys.argv[1:]))
+    except AssayError as error:
+        return _fail(error)

assay/compose.py

@@ -0,0 +1,31 @@
+"""One explicit dispatcher for Assay's closed composition methods."""
+
+from __future__ import annotations
+
+from typing import overload
+
+from assay.additive import additive
+from assay.contracts import AdditiveRequest, MinimumRequest, ScoreResult, WeightedMeanRequest
+from assay.minimum import minimum
+from assay.weighted_mean import weighted_mean
+
+
+@overload
+def compose(request: WeightedMeanRequest) -> ScoreResult: ...
+
+
+@overload
+def compose(request: AdditiveRequest) -> ScoreResult: ...
+
+
+@overload
+def compose(request: MinimumRequest) -> ScoreResult: ...
+
+
+def compose(request: WeightedMeanRequest | AdditiveRequest | MinimumRequest) -> ScoreResult:
+    """Dispatch one validated request without accepting executable formulas."""
+    if isinstance(request, WeightedMeanRequest):
+        return weighted_mean(request)
+    if isinstance(request, AdditiveRequest):
+        return additive(request)
+    return minimum(request)