reliability-lab 0.1.0a1__py3-none-any.whl

reliability_lab/__init__.py

@@ -0,0 +1,62 @@
+"""reliability-lab: study design for the reliability of LLM and human judgments."""
+
+from reliability_lab._version import __version__
+from reliability_lab.alpha import INTERVAL, NOMINAL, ORDINAL, RATIO, Distance
+from reliability_lab.data import RatingFrame, Status
+from reliability_lab.design import CoderSpec, Design
+from reliability_lab.exceptions import (
+    BootstrapWarning,
+    CategorySkewWarning,
+    CoderDependenceWarning,
+    DesignError,
+    DomainSizeWarning,
+    LeakageError,
+    ReliabilityLabError,
+    ReliabilityWarning,
+    SchemaError,
+    UndefinedAlphaError,
+)
+from reliability_lab.report import (
+    AlphaEstimate,
+    AlphaResult,
+    BootstrapResult,
+    ComparisonResult,
+    Coverage,
+    Interval,
+    Provenance,
+)
+from reliability_lab.split import SplitManifest
+from reliability_lab.study import Fold, ReliabilityStudy
+
+__all__ = [
+    "INTERVAL",
+    "NOMINAL",
+    "ORDINAL",
+    "RATIO",
+    "AlphaEstimate",
+    "AlphaResult",
+    "BootstrapResult",
+    "BootstrapWarning",
+    "CategorySkewWarning",
+    "CoderDependenceWarning",
+    "CoderSpec",
+    "ComparisonResult",
+    "Coverage",
+    "Design",
+    "DesignError",
+    "Distance",
+    "DomainSizeWarning",
+    "Fold",
+    "Interval",
+    "LeakageError",
+    "Provenance",
+    "RatingFrame",
+    "ReliabilityLabError",
+    "ReliabilityStudy",
+    "ReliabilityWarning",
+    "SchemaError",
+    "SplitManifest",
+    "Status",
+    "UndefinedAlphaError",
+    "__version__",
+]

reliability_lab/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0a1"

reliability_lab/alpha.py

@@ -0,0 +1,290 @@
+"""Krippendorff's alpha from coincidence matrices.
+
+Notation follows Krippendorff (2011), *Computing Krippendorff's alpha-reliability*:
+``o`` is the V x V coincidence matrix, ``n_c`` its marginals, ``n`` the number of
+pairable values and ``delta2`` the squared distances between values.
+
+    D_o   = sum(o * delta2) / n
+    D_e   = sum(n_c n_k delta2) / (n (n - 1))
+    alpha = 1 - D_o / D_e
+
+The coincidence matrix is a sum of per-unit contributions, and those do not depend
+on the distance. Precomputing them once therefore serves every distance and every
+bootstrap replicate. Only distances that depend on the marginals (ordinal) must be
+rebuilt from each replicate's ``n_c``.
+
+This module works on integer-coded numpy arrays only; ``study.py`` does the coding.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import numpy.typing as npt
+from scipy import sparse
+
+from reliability_lab.exceptions import DesignError
+
+FloatArray = npt.NDArray[np.float64]
+IntArray = npt.NDArray[np.intp]
+
+#: Above this many declared values a DomainSizeWarning is raised.
+DOMAIN_SIZE_WARN = 1_000
+#: Above this many declared values the study is refused.
+DOMAIN_SIZE_MAX = 10_000
+
+#: ``fn(values, marginals) -> V x V`` squared distances over the declared domain.
+DistanceFn = Callable[[npt.NDArray[Any], FloatArray], FloatArray]
+
+
+@dataclass(frozen=True, eq=False)
+class Distance:
+    """A squared-distance function delta² over a declared value domain.
+
+    ``fn`` receives the declared values (in declared order) and the marginal count
+    of each value, and returns a symmetric V x V matrix with a zero diagonal.
+    ``depends_on_marginals`` marks metrics such as ordinal whose distances change
+    with the category distribution. ``numeric`` marks metrics that need numbers.
+    """
+
+    name: str
+    fn: DistanceFn
+    depends_on_marginals: bool = False
+    numeric: bool = False
+
+    def matrix(self, values: Sequence[Any], marginals: npt.ArrayLike) -> FloatArray:
+        v = np.asarray(values, dtype=float if self.numeric else object)
+        m = np.asarray(marginals, dtype=float)
+        d = np.asarray(self.fn(v, m), dtype=float)
+        _validate_distance_matrix(d, len(values), self.name)
+        return d
+
+    @classmethod
+    def from_matrix(cls, name: str, matrix: npt.ArrayLike) -> Distance:
+        """A custom distance from a fixed V x V matrix of squared distances.
+
+        Rows and columns follow the order of the declared ``values``.
+        """
+        fixed = np.array(matrix, dtype=float)
+        if fixed.ndim != 2 or fixed.shape[0] != fixed.shape[1]:
+            raise DesignError(f"distance {name!r} must be a square matrix, got {fixed.shape}")
+        _validate_distance_matrix(fixed, fixed.shape[0], name)
+        fixed.setflags(write=False)
+
+        def fn(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
+            if len(values) != fixed.shape[0]:
+                raise DesignError(
+                    f"distance {name!r} is {fixed.shape[0]} x {fixed.shape[0]} but the value "
+                    f"domain has {len(values)} values"
+                )
+            return fixed
+
+        return cls(name=name, fn=fn)
+
+
+def _validate_distance_matrix(d: FloatArray, size: int, name: str) -> None:
+    if d.shape != (size, size):
+        raise DesignError(f"distance {name!r} returned shape {d.shape}, expected ({size}, {size})")
+    if not np.isfinite(d).all():
+        raise DesignError(f"distance {name!r} has non-finite entries")
+    if (d < 0).any():
+        raise DesignError(f"distance {name!r} has negative entries")
+    if not np.allclose(d, d.T, rtol=1e-12, atol=1e-12):
+        raise DesignError(f"distance {name!r} is not symmetric")
+    if np.any(np.diag(d) != 0):
+        raise DesignError(f"distance {name!r} must be zero on the diagonal")
+
+
+def _nominal(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
+    return 1.0 - np.eye(len(values))
+
+
+def _ordinal(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
+    # delta_ck = sum_{g=c..k} n_g - (n_c + n_k)/2, which equals r_k - r_c with
+    # r = cumsum(n) - n/2 (the mid-rank of each value in declared order).
+    r = np.cumsum(marginals) - marginals / 2.0
+    return np.asarray((r[:, None] - r[None, :]) ** 2, dtype=float)
+
+
+def _interval(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
+    v = values.astype(float)
+    return np.asarray((v[:, None] - v[None, :]) ** 2, dtype=float)
+
+
+def _ratio(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
+    v = values.astype(float)
+    if (v < 0).any():
+        raise DesignError("ratio distances need non-negative values")
+    total = v[:, None] + v[None, :]
+    diff = v[:, None] - v[None, :]
+    safe = np.where(total == 0, 1.0, total)
+    return np.asarray(np.where(total == 0, 0.0, (diff / safe) ** 2), dtype=float)
+
+
+NOMINAL = Distance("nominal", _nominal)
+ORDINAL = Distance("ordinal", _ordinal, depends_on_marginals=True)
+INTERVAL = Distance("interval", _interval, numeric=True)
+RATIO = Distance("ratio", _ratio, numeric=True)
+
+BUILTIN_DISTANCES: dict[str, Distance] = {d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO)}
+
+
+def check_domain_size(n_values: int) -> str | None:
+    """Refuse oversized domains; return a warning message for large ones."""
+    megabytes = n_values * n_values * 8 / 1e6
+    if n_values > DOMAIN_SIZE_MAX:
+        raise DesignError(
+            f"the value domain has {n_values} values (more than {DOMAIN_SIZE_MAX}); each "
+            f"V x V matrix would need about {megabytes:,.0f} MB"
+        )
+    if n_values > DOMAIN_SIZE_WARN:
+        return (
+            f"the value domain has {n_values} values; each V x V matrix needs about "
+            f"{megabytes:,.0f} MB"
+        )
+    return None
+
+
+def per_unit_coincidences(
+    unit_idx: IntArray, value_idx: IntArray, n_units: int, n_values: int
+) -> tuple[sparse.csr_array, FloatArray]:
+    """Each unit's contribution to the coincidence matrix.
+
+    Row ``u`` of the returned ``(n_units, n_values**2)`` sparse array is unit u's
+    V x V contribution, flattened in C order: ``(n_u n_u' - diag(n_u)) / (m_u - 1)``.
+    Here ``n_u`` counts the unit's pairable values and ``m_u`` is their total.
+    Units with fewer than two pairable values contribute nothing. Summing the rows
+    (or weighting them, in a bootstrap) gives the coincidence matrix.
+
+    Returns the sparse array and ``m_u`` for every unit.
+    """
+    unit_idx = np.asarray(unit_idx, dtype=np.intp)
+    value_idx = np.asarray(value_idx, dtype=np.intp)
+    counts = sparse.csr_array(
+        (np.ones(len(unit_idx)), (unit_idx, value_idx)), shape=(n_units, n_values)
+    )
+    counts.sum_duplicates()
+    counts.sort_indices()
+    pairable = np.asarray(counts.sum(axis=1), dtype=float).ravel()
+
+    indptr = counts.indptr.astype(np.intp)
+    cols = counts.indices.astype(np.intp)
+    data = counts.data.astype(float)
+    distinct = np.diff(indptr)
+    # Pair every non-zero entry with every non-zero entry of the same unit.
+    entry_unit = np.repeat(np.arange(n_units), distinct)
+    reps = distinct[entry_unit]
+    left = np.repeat(np.arange(len(cols)), reps)
+    block_start = np.repeat(np.cumsum(reps) - reps, reps)
+    right = np.repeat(indptr[entry_unit], reps) + (np.arange(len(left)) - block_start)
+
+    unit = entry_unit[left]
+    same = cols[left] == cols[right]
+    weight = data[left] * data[right] - np.where(same, data[left], 0.0)
+    keep = (pairable[unit] >= 2) & (weight != 0)
+    contribution = weight[keep] / (pairable[unit][keep] - 1)
+    flat = cols[left][keep] * n_values + cols[right][keep]
+    out = sparse.csr_array((contribution, (unit[keep], flat)), shape=(n_units, n_values * n_values))
+    return out, pairable
+
+
+def coincidence_matrix(per_unit: sparse.csr_array, n_values: int) -> FloatArray:
+    """Sum the per-unit contributions into the V x V coincidence matrix."""
+    flat = np.asarray(per_unit.sum(axis=0), dtype=float).ravel()
+    return flat.reshape(n_values, n_values)
+
+
+@dataclass(frozen=True, eq=False)
+class AlphaComponents:
+    """Alpha and the quantities it is computed from.
+
+    When alpha is undefined, ``alpha``, ``d_o`` and ``d_e`` hold what could be
+    computed (NaN otherwise) and ``undefined_reason`` says why. Nothing is raised
+    here, so bootstrap code can count failed replicates.
+    """
+
+    alpha: float
+    d_o: float
+    d_e: float
+    n: float
+    marginals: FloatArray
+    delta2: FloatArray
+    expected: FloatArray
+    undefined_reason: str | None
+
+    @property
+    def undefined(self) -> bool:
+        return self.undefined_reason is not None
+
+
+def alpha_from_coincidences(
+    coincidence: FloatArray, values: Sequence[Any], distance: Distance
+) -> AlphaComponents:
+    """Alpha from a coincidence matrix over the declared ``values``."""
+    o = np.asarray(coincidence, dtype=float)
+    n_c = o.sum(axis=1)
+    n = float(n_c.sum())
+    delta2 = distance.matrix(values, n_c)
+    if n < 2:
+        nan = float("nan")
+        return AlphaComponents(
+            nan,
+            nan,
+            nan,
+            n,
+            n_c,
+            delta2,
+            np.zeros_like(o),
+            "no unit has two or more pairable values",
+        )
+    expected = (np.outer(n_c, n_c) - np.diag(n_c)) / (n - 1)
+    d_o = float((o * delta2).sum() / n)
+    d_e = float((expected * delta2).sum() / n)
+    if not d_e > 0:
+        return AlphaComponents(
+            float("nan"),
+            d_o,
+            d_e,
+            n,
+            n_c,
+            delta2,
+            expected,
+            "expected disagreement is zero (every pairable value is in one category or at "
+            "zero distance), so alpha is undefined",
+        )
+    return AlphaComponents(1.0 - d_o / d_e, d_o, d_e, n, n_c, delta2, expected, None)
+
+
+@dataclass(frozen=True, eq=False)
+class AlphaComputation:
+    """Alpha for one set of pairable values, with its white-box decomposition.
+
+    ``unit_disagreement[u]`` is ``sum(o_u * delta2)``. Divided by ``n`` it is unit
+    u's contribution to D_o (the contributions sum to D_o). Divided by ``m_u`` it is
+    the mean squared distance over the unit's ordered pairs of values.
+    """
+
+    components: AlphaComponents
+    coincidence: FloatArray
+    per_unit: sparse.csr_array
+    pairable: FloatArray
+    unit_disagreement: FloatArray
+
+
+def compute_alpha(
+    unit_idx: IntArray,
+    value_idx: IntArray,
+    n_units: int,
+    values: Sequence[Any],
+    distance: Distance,
+) -> AlphaComputation:
+    """Alpha from integer-coded pairable values (one entry per rating)."""
+    n_values = len(values)
+    per_unit, pairable = per_unit_coincidences(unit_idx, value_idx, n_units, n_values)
+    o = coincidence_matrix(per_unit, n_values)
+    components = alpha_from_coincidences(o, values, distance)
+    unit_disagreement = np.asarray(per_unit @ components.delta2.ravel(), dtype=float)
+    return AlphaComputation(components, o, per_unit, pairable, unit_disagreement)

reliability_lab/data.py

@@ -0,0 +1,280 @@
+"""Long-form ratings: one row per rating attempt, each with an explicit status."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Hashable, Iterable
+from enum import StrEnum
+from typing import Any
+
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
+
+from reliability_lab.exceptions import SchemaError
+
+
+class Status(StrEnum):
+    """Outcome of one rating attempt."""
+
+    OBSERVED = "observed"
+    ABSTAINED = "abstained"
+    INAPPLICABLE = "inapplicable"
+    PARSE_ERROR = "parse_error"
+    PROVIDER_ERROR = "provider_error"
+
+
+#: Statuses of failed attempts, as plain strings (the stored form).
+ERROR_STATUSES: frozenset[str] = frozenset({Status.PARSE_ERROR.value, Status.PROVIDER_ERROR.value})
+
+#: Canonical field names, in the order a RatingFrame stores them.
+FIELDS: tuple[str, ...] = (
+    "unit_id",
+    "rater_id",
+    "value",
+    "status",
+    "group_id",
+    "stratum",
+    "model_id",
+    "model_family",
+    "condition_id",
+    "run_id",
+)
+
+
+def preview(labels: Iterable[Any], limit: int = 5) -> str:
+    """Format the first few labels of a collection for an error message."""
+    items = list(labels)
+    shown = ", ".join(repr(_plain(x)) for x in items[:limit])
+    more = f" and {len(items) - limit} more" if len(items) > limit else ""
+    return f"[{shown}]{more}"
+
+
+def _plain(x: Any) -> Any:
+    """Turn numpy scalars into Python scalars so they print cleanly."""
+    return x.item() if isinstance(x, np.generic) else x
+
+
+def canonical_text(x: Any) -> str:
+    """Stable text for fingerprints: missing is empty, integral floats print as ints."""
+    if x is None or (not isinstance(x, str) and bool(pd.isna(x))):
+        return ""
+    x = _plain(x)
+    if isinstance(x, float) and x.is_integer():
+        return str(int(x))
+    return str(x)
+
+
+def fingerprint_rows(columns: dict[str, Iterable[Any]]) -> str:
+    """sha256 over rows of named columns, independent of row order."""
+    names = list(columns)
+    texts = [[canonical_text(v) for v in col] for col in columns.values()]
+    rows = sorted("\x1f".join(parts) for parts in zip(*texts, strict=True))
+    digest = hashlib.sha256("\x1f".join(names).encode("utf-8"))
+    for row in rows:
+        digest.update(b"\n" + row.encode("utf-8"))
+    return digest.hexdigest()
+
+
+class RatingFrame:
+    """Validated long-form ratings.
+
+    Build one with :meth:`from_long`. Each row is one rating attempt by one rater
+    on one unit. Its ``status`` says whether a value was observed, the rater
+    abstained, the item was inapplicable, or the attempt failed (parse or provider
+    error). The frame is copied on construction and never mutated afterwards.
+    """
+
+    def __init__(self, data: pd.DataFrame, declared: frozenset[str], implicit_status: bool) -> None:
+        self._data = data
+        self._declared = declared
+        self._implicit_status = implicit_status
+        self._fingerprint: str | None = None
+
+    def fingerprint(self) -> str:
+        """sha256 of the validated ratings (declared fields only).
+
+        Rows are sorted before hashing, and values are written as canonical text,
+        so the fingerprint ignores row order and dtype changes such as ``1`` vs
+        ``1.0``.
+        """
+        if self._fingerprint is None:
+            fields = [f for f in FIELDS if f in self._declared]
+            self._fingerprint = fingerprint_rows({f: self._data[f] for f in fields})
+        return self._fingerprint
+
+    @classmethod
+    def from_long(
+        cls,
+        df: pd.DataFrame,
+        *,
+        unit: Hashable,
+        rater: Hashable,
+        value: Hashable,
+        status: Hashable | None = None,
+        group: Hashable | None = None,
+        stratum: Hashable | None = None,
+        model_id: Hashable | None = None,
+        model_family: Hashable | None = None,
+        condition_id: Hashable | None = None,
+        run_id: Hashable | None = None,
+    ) -> RatingFrame:
+        """Validate a long-form table and map its columns to canonical fields.
+
+        Each keyword names the column of ``df`` that holds that field. ``unit``,
+        ``rater`` and ``value`` are required. Without ``status`` every row must
+        carry a value and is recorded as observed. Pass a status column whenever
+        abstentions or failed attempts exist.
+        """
+        if not isinstance(df, pd.DataFrame):
+            raise TypeError(f"expected a pandas DataFrame, got {type(df).__name__}")
+        mapping: dict[str, Hashable | None] = {
+            "unit_id": unit,
+            "rater_id": rater,
+            "value": value,
+            "status": status,
+            "group_id": group,
+            "stratum": stratum,
+            "model_id": model_id,
+            "model_family": model_family,
+            "condition_id": condition_id,
+            "run_id": run_id,
+        }
+        absent = [c for c in mapping.values() if c is not None and c not in df.columns]
+        if absent:
+            raise SchemaError(f"columns not found in the ratings table: {preview(absent)}")
+
+        n = len(df)
+        columns: dict[str, Any] = {}
+        for field, column in mapping.items():
+            if column is None:
+                columns[field] = np.full(n, None, dtype=object)
+            else:
+                columns[field] = df[column].to_numpy(copy=True)
+        data = pd.DataFrame(columns, index=df.index)
+
+        for field in ("unit_id", "rater_id"):
+            _require_complete(data, field, mapping[field])
+
+        if status is None:
+            missing = data.index[data["value"].isna()]
+            if len(missing):
+                raise SchemaError(
+                    f"value is missing in {len(missing)} rows {preview(missing)}. Pass status= "
+                    "with a status column so that abstentions and failed attempts are explicit."
+                )
+            data["status"] = Status.OBSERVED.value
+        else:
+            data["status"] = _normalise_status(data["status"], status)
+            unobserved = data.index[(data["status"] == Status.OBSERVED) & data["value"].isna()]
+            if len(unobserved):
+                raise SchemaError(
+                    f"{len(unobserved)} rows have status 'observed' but no value "
+                    f"{preview(unobserved)}"
+                )
+
+        declared = frozenset(f for f, c in mapping.items() if c is not None)
+        for field in ("group_id", "stratum"):
+            if field in declared:
+                _require_complete(data, field, mapping[field])
+                _require_constant_within(data, field, "unit_id")
+
+        is_llm = data["model_id"].notna()
+        if is_llm.any():
+            if "condition_id" not in declared:
+                raise SchemaError(
+                    "LLM rows (non-null model_id) need condition_id: the frozen prompt, rubric, "
+                    "parser, candidate order and generation settings of the judge."
+                )
+            bad = data.index[is_llm & data["condition_id"].isna()]
+            if len(bad):
+                raise SchemaError(f"{len(bad)} LLM rows have no condition_id {preview(bad)}")
+            if "model_family" in declared:
+                _require_constant_within(data[is_llm], "model_family", "model_id")
+
+        return cls(data, declared | {"status"}, implicit_status=status is None)
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __repr__(self) -> str:
+        counts = self._data["status"].value_counts()
+        status = ", ".join(f"{k}={int(v)}" for k, v in counts.items())
+        return (
+            f"RatingFrame(rows={len(self)}, units={self.n_units}, "
+            f"raters={self._data['rater_id'].nunique()}, status: {status})"
+        )
+
+    @property
+    def declared(self) -> frozenset[str]:
+        """Canonical fields that were mapped to a column (``status`` always counts)."""
+        return self._declared
+
+    @property
+    def implicit_status(self) -> bool:
+        """True when no status column was given and every row was taken as observed."""
+        return self._implicit_status
+
+    @property
+    def n_units(self) -> int:
+        return int(self._data["unit_id"].nunique())
+
+    @property
+    def is_llm(self) -> npt.NDArray[np.bool_]:
+        """Mask of rows produced by an LLM judge (rows with a model_id)."""
+        return self._data["model_id"].notna().to_numpy()
+
+    def to_frame(self) -> pd.DataFrame:
+        """A copy of the validated table, with canonical column names."""
+        return self._data.copy()
+
+    def _subset(self, mask: npt.NDArray[np.bool_]) -> RatingFrame:
+        """The rows selected by a boolean mask. Every row and per-unit check that
+        held for the whole frame still holds for a subset, so nothing is re-run."""
+        return RatingFrame(self._data[mask].copy(), self._declared, self._implicit_status)
+
+    def observed_values(self) -> list[Any]:
+        """Distinct observed values, sorted when they are mutually comparable.
+
+        Useful for declaring a value domain explicitly:
+        ``values=ratings.observed_values()``.
+        """
+        observed = self._data.loc[self._data["status"] == Status.OBSERVED, "value"]
+        unique = [_plain(v) for v in pd.unique(observed)]
+        try:
+            return sorted(unique)
+        except TypeError:
+            return unique
+
+
+def _require_complete(data: pd.DataFrame, field: str, column: Hashable | None) -> None:
+    missing = data.index[data[field].isna()]
+    if len(missing):
+        raise SchemaError(
+            f"{field} (column {column!r}) is missing in {len(missing)} rows {preview(missing)}"
+        )
+
+
+def _require_constant_within(data: pd.DataFrame, field: str, key: str) -> None:
+    per_key = data.groupby(key, sort=False)[field].nunique()
+    bad = per_key.index[per_key > 1]
+    if len(bad):
+        raise SchemaError(
+            f"each {key} must have exactly one {field}; {len(bad)} have several {preview(bad)}"
+        )
+
+
+def _normalise_status(raw: pd.Series[Any], column: Hashable) -> pd.Series[Any]:
+    missing = raw.index[raw.isna()]
+    if len(missing):
+        raise SchemaError(
+            f"status (column {column!r}) is missing in {len(missing)} rows {preview(missing)}"
+        )
+    labels = raw.map(str)
+    legal = {s.value for s in Status}
+    unknown = sorted(set(labels[~labels.isin(legal)]))
+    if unknown:
+        raise SchemaError(
+            f"unknown status labels {preview(unknown)}; expected one of {sorted(legal)}"
+        )
+    return labels.astype(object)