interrater 0.0.1.dev0__py3-none-any.whl

interrater/data_objects/_ratings_validation.py

@@ -0,0 +1,160 @@
+"""
+Validation for Ratings.
+
+Everything here raises. Semantics declares what a study should look like but
+never sees an array; Ratings holds both, so this is where a declaration meets
+the data that is supposed to satisfy it.
+
+Kept apart from the arithmetic so that a bootstrapper can import the maths
+without dragging error machinery along.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from ..contracts import EMPTY, MIN_SUBJECTS
+from ..semantics.ratings_semantics import RatingsSemantics
+
+_P = "Ratings"
+
+
+# ---- The array on its own ---- #
+
+
+def validate_dtype(category_index: np.ndarray) -> None:
+    if category_index.dtype.kind == "f":
+        raise TypeError(
+            f"{_P}.category_index must be an integer array, got dtype "
+            f"{category_index.dtype}. Floats usually mean NaN was used for "
+            f"absent ratings; use EMPTY ({EMPTY}) instead, which keeps the "
+            f"array integral and indexable."
+        )
+    if category_index.dtype.kind not in "iu":
+        raise TypeError(
+            f"{_P}.category_index must be an integer array, got dtype "
+            f"{category_index.dtype}. Category names are resolved to positions "
+            f"by the cleaner; by the time an array reaches me it holds "
+            f"integers."
+        )
+
+
+def validate_shape(category_index: np.ndarray) -> None:
+    if category_index.ndim != 2:
+        raise ValueError(
+            f"{_P}.category_index must be 2-dimensional "
+            f"(n_subjects, n_raters), got shape {category_index.shape}."
+        )
+    if category_index.shape[0] < MIN_SUBJECTS:
+        raise ValueError(
+            f"{_P} needs at least {MIN_SUBJECTS} subjects, got "
+            f"{category_index.shape[0]}."
+        )
+
+
+# ---- The array against the declaration ---- #
+
+
+def validate_matches_raters(
+    category_index: np.ndarray, semantics: RatingsSemantics
+) -> None:
+    """The check that makes declaring a study worth the trouble."""
+    found = category_index.shape[1]
+    if found != semantics.n_raters:
+        raise ValueError(
+            f"{_P} was given {found} rater column(s) but its semantics declare "
+            f"{semantics.n_raters}: {list(semantics.raters)}. A column that "
+            f"was not declared is a mistake to look into, not a rater to "
+            f"invent; if the extra column is real, add it to the semantics."
+        )
+
+
+def validate_matches_n_subjects(
+    category_index: np.ndarray, semantics: RatingsSemantics
+) -> None:
+    if semantics.n_subjects is None:
+        return
+    found = category_index.shape[0]
+    if found != semantics.n_subjects:
+        raise ValueError(
+            f"{_P} was given {found:,} subject row(s) but its semantics expect "
+            f"{semantics.n_subjects:,}. Either the file is not the one the "
+            f"study describes, or a cleaner dropped rows it should not have. "
+            f"Set n_subjects to None in the semantics for no expectation."
+        )
+
+
+def validate_values_in_categories(
+    category_index: np.ndarray, semantics: RatingsSemantics
+) -> None:
+    k       = semantics.n_categories
+    illegal = (category_index < EMPTY) | (category_index >= k)
+    n_bad   = int(illegal.sum())
+    if n_bad:
+        examples = np.unique(category_index[illegal])[:3].tolist()
+        raise ValueError(
+            f"{_P}.category_index holds {n_bad:,} value(s) outside its "
+            f"categories (e.g. {examples}). Valid values are 0..{k - 1}, "
+            f"indexing {list(semantics.categories.names)}, or EMPTY ({EMPTY}). "
+            f"Use a cleaner to map raw responses onto the categories first."
+        )
+
+
+# ---- Subject ids ---- #
+
+
+def validate_subject_ids_match_shape(
+    category_index: np.ndarray, subject_ids: np.ndarray
+) -> None:
+    if len(subject_ids) != category_index.shape[0]:
+        raise ValueError(
+            f"{_P} has {category_index.shape[0]:,} subject row(s) but "
+            f"{len(subject_ids):,} subject id(s). Every row needs a label."
+        )
+
+
+def validate_subject_ids_unique(
+    subject_ids: np.ndarray, semantics: RatingsSemantics
+) -> None:
+    if not semantics.unique_subjects:
+        return
+    uniques, counts = np.unique(subject_ids, return_counts=True)
+    repeated = uniques[counts > 1]
+    if repeated.size:
+        raise ValueError(
+            f"{_P} has {repeated.size:,} subject id(s) appearing more than "
+            f"once (e.g. {repeated[:3].tolist()}). That is usually an export "
+            f"that duplicated rows. If the study deliberately re-shows "
+            f"subjects, set unique_subjects=False in the semantics."
+        )
+
+
+# ---- Nothing may be entirely absent ---- #
+
+
+def validate_no_idle_raters(
+    category_index: np.ndarray, semantics: RatingsSemantics
+) -> None:
+    rated = (category_index != EMPTY).sum(axis=0)
+    idle  = np.flatnonzero(rated == 0)
+    if idle.size:
+        names = [semantics.raters[i] for i in idle[:3]]
+        raise ValueError(
+            f"{_P} has {idle.size} declared rater(s) who rated nothing at all "
+            f"(e.g. {names}). A rater with no ratings contributes to no "
+            f"coefficient and makes per-rater rates undefined. Drop them from "
+            f"the semantics, or find out why their column is empty."
+        )
+
+
+def validate_no_unrated_subjects(
+    category_index: np.ndarray, subject_ids: np.ndarray
+) -> None:
+    rated   = (category_index != EMPTY).sum(axis=1)
+    unrated = np.flatnonzero(rated == 0)
+    if unrated.size:
+        raise ValueError(
+            f"{_P} has {unrated.size:,} subject(s) rated by nobody (e.g. "
+            f"{subject_ids[unrated][:3].tolist()}). An all-empty row joins no "
+            f"comparison; drop these in the cleaner."
+        )

interrater/data_objects/ratings.py

@@ -0,0 +1,188 @@
+"""The data object: a validated array of ratings and what it means."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+
+from ..contracts import INDEX_DTYPE, LABEL_DTYPE
+from ..semantics.ratings_semantics import RatingsSemantics
+from . import _ratings_validation as _v
+
+
+class Ratings:
+    """
+    I am a validated array of ratings. If I exist, I am complete.
+
+    I hold three things: a dense ``(n_subjects, n_raters)`` array of category
+    positions, a label for each row, and the semantics that say what all of it
+    means. That is the whole of me. I do not clean, analyse, transform, or
+    count -- a numpy array does not know what it is, and my job is to be the
+    array that does.
+
+    My constructor raises rather than repair. Anything holding me may assume my
+    values are in range, my shape matches what my semantics declare, and no
+    declared rater of mine sat idle.
+
+    I describe my contents, not my history. Nothing about what a cleaner
+    discarded to make me survives in me; that lives in its report, alongside
+    the config that produced it.
+
+    I compute nothing about myself either. How skewed my categories are,
+    whether every subject was rated the same number of times, which raters
+    overlap -- those are facts *about* me, and they belong to a
+    ``RatingsProfile``. Keeping them out means I stay cheap to build, which
+    matters when a bootstrapper is making thousands of arrays a second.
+
+    I allow empties freely. Raters need not have rated the same subjects, which
+    is the ordinary case in a systematic review where each subject goes to a
+    subset of reviewers. An empty says only that no rating exists there, never
+    why: 'unsure' and 'abstain' are categories a study designer offered and a
+    rater chose, and they count like any other answer.
+
+    Attributes
+    ----------
+    category_index:
+        (n_subjects, n_raters) int8, read-only. Each cell is a position in
+        ``semantics.categories.names``, or EMPTY. Column order is rater order
+        as declared; I never sort.
+    subject_ids:
+        (n_subjects,) object. One label per row, in input order. Generated as
+        ``subject0, subject1, ...`` when the data had none, since a row must be
+        nameable even if the file did not name it. May repeat only if the
+        semantics allow it.
+    semantics:
+        Who the raters are, what the categories mean, and what the data was
+        expected to look like.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from interrater.semantics import RatingsSemantics, UnorderedCategories
+    >>> sem = RatingsSemantics(
+    ...     raters=["ann", "bob"],
+    ...     categories=UnorderedCategories(("include", "exclude")),
+    ... )
+    >>> r = Ratings(np.array([[0, 0], [1, 1], [0, 1]], dtype=np.int8), sem)
+    >>> r.n_subjects, r.n_raters
+    (3, 2)
+    >>> r.subject_ids[0]
+    'subject0'
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    category_index: np.ndarray
+    # (n_subjects, n_raters) int8, C-contiguous, read-only after construction.
+    # A cell is a position in semantics.categories.names, or EMPTY (-1).
+    # K is owned by the semantics and is never inferred from these values:
+    # category_index.max() + 1 is only ever a lower bound on K.
+
+    subject_ids:    np.ndarray
+    # (n_subjects,) object, holding real Python strings so long ids survive.
+    # Row order is meaningful -- it is what distinguishes a bootstrap draw from
+    # the sample it came from.
+
+    semantics:      RatingsSemantics
+    # The declaration my contents were checked against.
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        category_index: np.ndarray,
+        semantics:      RatingsSemantics,
+        subject_ids:    Sequence[object] | None = None,
+    ) -> None:
+        self.category_index = np.ascontiguousarray(category_index)
+        self.semantics      = semantics
+
+        _v.validate_dtype(self.category_index)
+        self.category_index = self.category_index.astype(INDEX_DTYPE, copy=False)
+        _v.validate_shape(self.category_index)
+
+        self.subject_ids = self._as_subject_ids(
+            subject_ids, self.category_index.shape[0]
+        )
+
+        self._validate()
+        self._freeze()
+
+    @staticmethod
+    def _as_subject_ids(
+        subject_ids: Sequence[object] | None, n_rows: int
+    ) -> np.ndarray:
+        """Absent ids become positional ones; given ids are taken as given."""
+        if subject_ids is None:
+            return np.array(
+                [f"subject{i}" for i in range(n_rows)], dtype=LABEL_DTYPE
+            )
+        return np.asarray(subject_ids, dtype=LABEL_DTYPE)
+
+    def _validate(self) -> None:
+        """Every check I make, in order."""
+        _v.validate_matches_raters(self.category_index, self.semantics)
+        _v.validate_matches_n_subjects(self.category_index, self.semantics)
+        _v.validate_values_in_categories(self.category_index, self.semantics)
+        _v.validate_subject_ids_match_shape(self.category_index, self.subject_ids)
+        _v.validate_subject_ids_unique(self.subject_ids, self.semantics)
+        _v.validate_no_idle_raters(self.category_index, self.semantics)
+        _v.validate_no_unrated_subjects(self.category_index, self.subject_ids)
+
+    def _freeze(self) -> None:
+        """Read-only, so nothing computed from me can be invalidated later."""
+        self.category_index = self.category_index.view()
+        self.category_index.flags.writeable = False
+        self.subject_ids = self.subject_ids.view()
+        self.subject_ids.flags.writeable = False
+
+    # ---------------------------------------------------------------- Shape #
+
+    @property
+    def n_subjects(self) -> int:
+        return int(self.category_index.shape[0])
+
+    @property
+    def n_raters(self) -> int:
+        return int(self.category_index.shape[1])
+
+    @property
+    def n_categories(self) -> int:
+        """K, from the semantics. Never inferred from my values."""
+        return self.semantics.n_categories
+
+    # ------------------------------------------------------------- Equality #
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Two of me are equal when we hold the same ratings and mean the same
+        thing.
+
+        Contents only. Where the data came from is not part of the comparison,
+        because I do not carry that. Positional on both axes: rater order is
+        declared order, and subject order distinguishes a bootstrap draw from
+        the sample it came from.
+        """
+        if not isinstance(other, Ratings):
+            return NotImplemented
+        return (
+            self.semantics == other.semantics
+            and self.category_index.shape == other.category_index.shape
+            and np.array_equal(self.category_index, other.category_index)
+            and np.array_equal(self.subject_ids, other.subject_ids)
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        filled = int((self.category_index != -1).sum())
+        total  = self.n_subjects * self.n_raters
+        return (
+            f"Ratings(\n"
+            f"    n_subjects = {self.n_subjects:,}\n"
+            f"    raters     = {list(self.semantics.raters)}\n"
+            f"    categories = {list(self.semantics.categories.names)}\n"
+            f"    filled     = {filled:,} / {total:,} ({filled / total:.1%})\n"
+            f")"
+        )

interrater/semantics/__init__.py

@@ -0,0 +1,19 @@
+from .categories import (
+    Categories,
+    DistanceCategories,
+    OrderedCategories,
+    UnorderedCategories,
+    WeightedCategories,
+    build_categories,
+)
+from .ratings_semantics import RatingsSemantics
+
+__all__ = [
+    "Categories",
+    "UnorderedCategories",
+    "DistanceCategories",
+    "OrderedCategories",
+    "WeightedCategories",
+    "build_categories",
+    "RatingsSemantics",
+]

interrater/semantics/categories.py

@@ -0,0 +1,327 @@
+"""What a rater was allowed to say, and how far apart those answers are."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+_MIN_CATEGORIES = 2
+
+
+class Categories(ABC):
+    """
+    I am the set of responses a rater was allowed to give.
+
+    I am the study design, not a summary of the data. I list what was offered,
+    never what was chosen. If I name five grades and nobody earned a D, I still
+    have five categories, because the raters could have said D and did not --
+    and any coefficient whose chance term divides by the number of categories
+    would move if I quietly shrank.
+
+    I own K. Nothing else in the package may infer it: the largest code present
+    in a dataset is a lower bound on K and never K itself, and that gap is
+    exactly where a bootstrap draw goes wrong when a rare category happens to
+    miss a resample.
+
+    I hold no ratings and do no counting. My position in a list is the integer
+    a ratings array stores, so my order is my encoding.
+
+    Attributes
+    ----------
+    names:
+        The allowed responses, in encoding order. Position i is code i.
+    labels:
+        Optional display names. Cosmetic; never used in arithmetic.
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    names:  tuple[str, ...]
+    labels: dict[str, str]
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        names:  object,
+        labels: dict[str, str] | None = None,
+    ) -> None:
+        self.names  = tuple(names)
+        self.labels = dict(labels or {})
+        self._validate()
+
+    def _validate(self) -> None:
+        """Every check I make. Subclasses extend, never replace."""
+        name = type(self).__name__
+
+        if len(self.names) < _MIN_CATEGORIES:
+            raise ValueError(
+                f"{name} needs at least {_MIN_CATEGORIES} categories, got "
+                f"{len(self.names)}. A single category cannot produce "
+                f"agreement or disagreement."
+            )
+
+        seen: set[str] = set()
+        dupes = [n for n in self.names if n in seen or seen.add(n)]
+        if dupes:
+            raise ValueError(
+                f"{name}.names contains {len(dupes)} duplicate entr(ies) "
+                f"(e.g. {dupes[:3]}). Position is the integer encoding, so a "
+                f"duplicate would make two codes mean the same thing."
+            )
+
+        unknown = sorted(set(self.labels) - set(self.names))
+        if unknown:
+            raise ValueError(
+                f"{name}.labels names {len(unknown)} categor(ies) that are not "
+                f"in this set (e.g. {unknown[:3]}). Add them to names, or drop "
+                f"them from labels."
+            )
+
+    # ------------------------------------------------------------ Public API #
+
+    @property
+    def n_categories(self) -> int:
+        """K. What the raters could have said, not what they did say."""
+        return len(self.names)
+
+    def index_of(self, category: str) -> int:
+        """The integer a ratings array stores for ``category``."""
+        try:
+            return self.names.index(category)
+        except ValueError:
+            raise KeyError(
+                f"{type(self).__name__} has no category {category!r}. Known: "
+                f"{list(self.names)}."
+            ) from None
+
+    def label_for(self, category: str) -> str:
+        """Display name, falling back to the category itself."""
+        return self.labels.get(category, category)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Categories):
+            return NotImplemented
+        return type(self) is type(other) and self.names == other.names
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        return (
+            f"{type(self).__name__}(\n"
+            f"    n_categories = {self.n_categories}\n"
+            f"    names        = {list(self.names)}\n"
+            f")"
+        )
+
+
+class UnorderedCategories(Categories):
+    """
+    I am categories with no ranking and no notion of distance.
+
+    Nominal. 'include', 'unsure' and 'exclude' are three different answers, and
+    nothing in me says 'unsure' sits between the other two -- even when a human
+    reading the words would assume it does. If you want that assumption in the
+    arithmetic, say so with OrderedCategories.
+
+    Every disagreement I describe is equally wrong, which is what unweighted
+    kappa already computes. A weighted coefficient handed me would reduce to
+    the unweighted one, so analysers that need distance should refuse me rather
+    than silently return the same number under a different name.
+    """
+
+    names:  tuple[str, ...]
+    labels: dict[str, str]
+
+
+class DistanceCategories(Categories, ABC):
+    """
+    I am categories that know how wrong each disagreement is.
+
+    My children disagree about where that knowledge comes from -- derived from
+    rank, or supplied outright -- but both can answer the same question, so a
+    weighted coefficient can take either and need not care which.
+
+    I exist because that question has exactly one form: a K x K matrix of
+    agreement weights, 1.0 on the diagonal, falling towards 0.0 as two
+    categories grow further apart.
+    """
+
+    @abstractmethod
+    def weight_matrix(self) -> np.ndarray:
+        """
+        (K, K) float. Agreement weight between every pair of categories.
+
+        1.0 means indistinguishable, 0.0 means maximally different. The
+        diagonal is 1.0 and the matrix is symmetric: how wrong a disagreement
+        is does not depend on which rater said which.
+        """
+
+
+class OrderedCategories(DistanceCategories):
+    """
+    I am categories on a ranked scale, and I derive distance from position.
+
+    'mild' < 'moderate' < 'severe'. My order is not merely an encoding, it is a
+    claim: adjacent categories are more alike than distant ones, so a reviewer
+    who says 'mild' where another said 'moderate' has disagreed less than one
+    who said 'severe'.
+
+    I compute weights from rank distance, either linearly or quadratically.
+    Quadratic is the common default in the medical literature and forgives
+    near-misses more readily; linear penalises distance evenly. Neither is more
+    correct, so I make you say which -- the choice changes the coefficient and
+    ought to appear in a methods section.
+
+    Attributes
+    ----------
+    weighting:
+        'linear' or 'quadratic'. How rank distance becomes disagreement.
+    """
+
+    names:     tuple[str, ...]
+    labels:    dict[str, str]
+    weighting: str
+
+    _ALLOWED = ("linear", "quadratic")
+
+    def __init__(
+        self,
+        names:     object,
+        weighting: str = "quadratic",
+        labels:    dict[str, str] | None = None,
+    ) -> None:
+        self.weighting = weighting
+        super().__init__(names, labels)
+
+    def _validate(self) -> None:
+        super()._validate()
+        if self.weighting not in self._ALLOWED:
+            raise ValueError(
+                f"{type(self).__name__}.weighting must be one of "
+                f"{list(self._ALLOWED)}, got {self.weighting!r}. Linear "
+                f"penalises rank distance evenly; quadratic forgives "
+                f"near-misses. The choice changes the coefficient."
+            )
+
+    def weight_matrix(self) -> np.ndarray:
+        k        = self.n_categories
+        rank     = np.arange(k)
+        distance = np.abs(rank[:, None] - rank[None, :]).astype(np.float64)
+        if self.weighting == "quadratic":
+            distance = distance ** 2
+        return 1.0 - distance / distance.max()
+
+
+class WeightedCategories(DistanceCategories):
+    """
+    I am categories with distances supplied outright, not inferred.
+
+    I do not require a ranking. 'cat' and 'dog' may be closer to each other
+    than either is to 'car' without any of the three being greater or less than
+    another, and no ordered scale can express that.
+
+    Use me when the similarity you mean cannot be recovered from an order:
+    grouped diagnoses, a coding frame where two labels are near-synonyms, or a
+    scale whose steps are genuinely uneven. Otherwise OrderedCategories is
+    fewer numbers to get wrong.
+
+    Attributes
+    ----------
+    weights:
+        (K, K) float. Symmetric, 1.0 on the diagonal, entries in [0, 1].
+    """
+
+    names:   tuple[str, ...]
+    labels:  dict[str, str]
+    weights: np.ndarray
+
+    def __init__(
+        self,
+        names:   object,
+        weights: np.ndarray,
+        labels:  dict[str, str] | None = None,
+    ) -> None:
+        self.weights = np.asarray(weights, dtype=np.float64)
+        super().__init__(names, labels)
+
+    def _validate(self) -> None:
+        super()._validate()
+        name = type(self).__name__
+        k    = self.n_categories
+
+        if self.weights.shape != (k, k):
+            raise ValueError(
+                f"{name}.weights must be ({k}, {k}) to match its {k} "
+                f"categories, got {self.weights.shape}."
+            )
+        if not np.allclose(self.weights, self.weights.T):
+            raise ValueError(
+                f"{name}.weights must be symmetric. How wrong a disagreement "
+                f"is cannot depend on which rater said which."
+            )
+        if not np.allclose(np.diag(self.weights), 1.0):
+            raise ValueError(
+                f"{name}.weights must have 1.0 on the diagonal: a category "
+                f"agrees perfectly with itself. Got "
+                f"{np.diag(self.weights).round(3).tolist()}."
+            )
+        if self.weights.min() < 0.0 or self.weights.max() > 1.0:
+            raise ValueError(
+                f"{name}.weights must lie in [0, 1], where 1.0 is "
+                f"indistinguishable and 0.0 maximally different. Got range "
+                f"[{self.weights.min():.3f}, {self.weights.max():.3f}]. If you "
+                f"have distances rather than similarities, subtract them from "
+                f"1 after scaling to [0, 1]."
+            )
+
+    def weight_matrix(self) -> np.ndarray:
+        return self.weights
+
+
+# ---------------------------------------------------------------- Factory #
+
+_KINDS = {
+    "unordered": UnorderedCategories,
+    "ordered":   OrderedCategories,
+    "weighted":  WeightedCategories,
+}
+
+
+def build_categories(spec: dict) -> Categories:
+    """
+    Build the right Categories from a plain mapping, as parsed from YAML.
+
+    ``kind`` selects the class; the remaining keys are its arguments. Kept here
+    rather than with the config that calls it, because this is the module that
+    knows what subclasses exist.
+
+    Examples
+    --------
+    >>> build_categories({"kind": "unordered", "names": ["yes", "no"]})
+    UnorderedCategories(
+        n_categories = 2
+        names        = ['yes', 'no']
+    )
+    """
+    spec = dict(spec)
+    kind = spec.pop("kind", None)
+    if kind not in _KINDS:
+        raise ValueError(
+            f"categories.kind must be one of {sorted(_KINDS)}, got {kind!r}. "
+            f"Use 'unordered' for a nominal scheme, 'ordered' for a ranked "
+            f"scale, or 'weighted' to supply a similarity matrix directly."
+        )
+    if "names" not in spec:
+        raise KeyError(
+            f"categories of kind {kind!r} needs 'names': the responses a rater "
+            f"was allowed to give, in encoding order."
+        )
+    try:
+        return _KINDS[kind](**spec)
+    except TypeError as error:
+        raise TypeError(
+            f"categories of kind {kind!r} got unexpected keys "
+            f"{sorted(set(spec) - {'names'})}. {error}"
+        ) from None