interrater 0.0.1.dev0__py3-none-any.whl

interrater/cleaners/drop_report.py

@@ -0,0 +1,159 @@
+"""What was removed for having too little data, and in what order."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class DropPass:
+    """One sweep of the drop loop."""
+
+    number:     int
+    n_raters:   int
+    n_subjects: int
+
+    @property
+    def dropped_nothing(self) -> bool:
+        return self.n_raters == 0 and self.n_subjects == 0
+
+
+@dataclass(frozen=True)
+class DropReport:
+    """
+    I record what was removed for thinness, and how the removing went.
+
+    Dropping is not symmetric: taking out a rater can push a subject below its
+    threshold and taking out a subject can push a rater below its own. So the
+    order was a declared choice, and I record which one was made along with
+    what it cost.
+
+    Under ``iterate`` I keep every pass separately, because the sequence is the
+    diagnostic. A study going 1,617 to 1,580 to 1,204 to 340 has cascaded, and
+    that is a very different event from a single clean cut to 340 -- yet both
+    end at the same number and a total alone cannot tell them apart.
+
+    I also record how the loop ended. Converging means the data settled and the
+    result is what the policy asked for. Hitting the pass limit means the
+    cascade was still running when it was cut off, and what survived is
+    arbitrary rather than principled.
+
+    Attributes
+    ----------
+    method:
+        Which order was used: raters first, subjects first, or iterate.
+    passes:
+        The sweeps that ran. One entry for the non-iterating methods.
+    converged:
+        Whether the loop stopped because a pass dropped nothing. False means it
+        hit the limit with work still to do, and the result should be treated
+        with suspicion.
+    dropped_raters:
+        Rater name to how many ratings that rater had when removed.
+    dropped_subject_ids:
+        Every subject removed, for the full rendering. Summarised as a
+        distribution otherwise.
+    dropped_subject_distribution:
+        How many subjects were removed holding how many ratings:
+        ``{0: 12, 1: 380}``.
+    """
+
+    method:    str
+    converged: bool = True
+    passes:    tuple[DropPass, ...] = ()
+
+    dropped_raters:      dict[str, int] = field(default_factory=dict)
+    dropped_subject_ids: tuple[str, ...] = ()
+    dropped_subject_distribution: dict[int, int] = field(default_factory=dict)
+
+    @property
+    def n_passes(self) -> int:
+        return len(self.passes)
+
+    @property
+    def n_raters_dropped(self) -> int:
+        return len(self.dropped_raters)
+
+    @property
+    def n_subjects_dropped(self) -> int:
+        return len(self.dropped_subject_ids)
+
+    @property
+    def dropped_nothing(self) -> bool:
+        return not self.dropped_raters and not self.dropped_subject_ids
+
+    def summary(self) -> dict:
+        return {
+            "method":    self.method,
+            "converged": self.converged,
+            "n_passes":  self.n_passes,
+            "passes": [
+                {"pass": p.number, "raters": p.n_raters, "subjects": p.n_subjects}
+                for p in self.passes
+            ],
+            "n_raters_dropped":   self.n_raters_dropped,
+            "n_subjects_dropped": self.n_subjects_dropped,
+            "dropped_raters":      dict(self.dropped_raters),
+            "dropped_subject_ids": list(self.dropped_subject_ids),
+            "dropped_subject_distribution": {
+                int(k): int(v)
+                for k, v in self.dropped_subject_distribution.items()
+            },
+        }
+
+    def render(self, verbosity: str = "summary") -> str:
+        pad = "    "
+
+        if self.dropped_nothing:
+            return f"Drops\n{pad}nothing dropped ({self.method})"
+
+        ending = (
+            f"converged after {self.n_passes} pass(es)"
+            if self.converged
+            else f"STOPPED AT THE PASS LIMIT after {self.n_passes}; "
+                 f"the cascade had not settled"
+        )
+        lines = [
+            "Drops",
+            f"{pad}method     : {self.method} ({ending})",
+            f"{pad}raters     : {self.n_raters_dropped} removed",
+        ]
+        if self.dropped_raters:
+            shown = ", ".join(
+                f"{name} ({n:,} rating(s))"
+                for name, n in list(self.dropped_raters.items())[:3]
+            )
+            more = self.n_raters_dropped - min(3, self.n_raters_dropped)
+            lines.append(
+                f"{pad}             {shown}"
+                + (f", +{more} more" if more > 0 else "")
+            )
+        lines.append(f"{pad}subjects   : {self.n_subjects_dropped:,} removed")
+
+        if self.dropped_subject_distribution:
+            shown = ", ".join(
+                f"{n:,} held {k}"
+                for k, n in sorted(self.dropped_subject_distribution.items())
+            )
+            lines.append(f"{pad}             {shown}")
+
+        if verbosity == "full":
+            if len(self.passes) > 1:
+                for p in self.passes:
+                    lines.append(
+                        f"{pad}  pass {p.number}: -{p.n_raters} rater(s), "
+                        f"-{p.n_subjects:,} subject(s)"
+                    )
+            if self.dropped_subject_ids:
+                ids = list(self.dropped_subject_ids)
+                shown = ", ".join(map(str, ids[:10]))
+                more  = len(ids) - min(10, len(ids))
+                lines.append(
+                    f"{pad}  ids: {shown}"
+                    + (f", +{more:,} more" if more > 0 else "")
+                )
+
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:
+        return self.render()

interrater/cleaners/frame_layout.py

@@ -0,0 +1,102 @@
+"""How to read the frame, as opposed to what the study means."""
+
+from __future__ import annotations
+
+
+class FrameLayout:
+    """
+    I describe the file, not the study.
+
+    Everything else a cleaner is configured with -- which strings mean nothing,
+    what to do with an undeclared response, how thin the data may get -- is a
+    decision about the research and travels unchanged to a second export. I am
+    the part that does not: which column holds the subject ids, whether the
+    header row can be trusted. A different spreadsheet of the same study needs
+    a different me.
+
+    I take a frame, never a path. Reading the file is the caller's job, and by
+    the time a frame reaches a cleaner the question of whether the file had a
+    header has already been answered -- pandas assigns integer labels when it
+    did not, so nothing downstream could tell the difference anyway. What I
+    decide is narrower: whether to trust the labels that are there.
+
+    Attributes
+    ----------
+    use_column_names:
+        True means the frame's column labels must equal the declared raters, in
+        order, and anything else is an error. A column nobody declared is a
+        mistake to look into, not a rater to invent. False ignores the labels
+        entirely and takes rater names from the semantics by position.
+    subject_id_col:
+        Which column holds subject ids, or None to generate them positionally.
+        Requires ``use_column_names``: there is nothing to find a named column
+        by otherwise.
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    use_column_names: bool
+    subject_id_col:   str | None
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        use_column_names: bool = True,
+        subject_id_col:   str | None = None,
+    ) -> None:
+        self.use_column_names = use_column_names
+        self.subject_id_col   = subject_id_col
+        self._validate()
+
+    def _validate(self) -> None:
+        name = type(self).__name__
+
+        if not isinstance(self.use_column_names, bool):
+            raise TypeError(
+                f"{name}.use_column_names must be a boolean, got "
+                f"{type(self.use_column_names).__name__}."
+            )
+
+        if self.subject_id_col is None:
+            return
+
+        if not isinstance(self.subject_id_col, str):
+            raise TypeError(
+                f"{name}.subject_id_col must be a column name or None, got "
+                f"{type(self.subject_id_col).__name__}."
+            )
+
+        if not self.use_column_names:
+            raise ValueError(
+                f"{name} sets subject_id_col to {self.subject_id_col!r} but "
+                f"use_column_names is False, so there are no column names to "
+                f"find it by. Either trust the labels, or drop subject_id_col "
+                f"and let ids be generated."
+            )
+
+    @classmethod
+    def from_dict(cls, spec: dict) -> FrameLayout:
+        unknown = sorted(set(spec) - {"use_column_names", "subject_id_col"})
+        if unknown:
+            raise KeyError(
+                f"frame_layout has unknown key(s) {unknown}. Known: "
+                f"use_column_names, subject_id_col."
+            )
+        return cls(**spec)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, FrameLayout):
+            return NotImplemented
+        return (
+            self.use_column_names == other.use_column_names
+            and self.subject_id_col == other.subject_id_col
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        return (
+            f"FrameLayout(use_column_names={self.use_column_names}, "
+            f"subject_id_col={self.subject_id_col!r})"
+        )

interrater/cleaners/insufficient_data_policy.py

@@ -0,0 +1,123 @@
+"""How thin the data may get before rows and columns are dropped."""
+
+from __future__ import annotations
+
+from ..contracts import MIN_RATINGS_PER_RATER, MIN_RATINGS_PER_SUBJECT
+
+MAX_DROP_PASSES: int = 100
+"""Safety rail on ``method: iterate``.
+
+Each pass only removes rows and columns, so iteration always converges and
+normally stops after two or three. Reaching this limit means something
+pathological is happening, and a result truncated mid-cascade is arbitrary --
+which is why the report says which way the loop ended rather than only how many
+passes it took.
+"""
+
+
+class InsufficientDataPolicy:
+    """
+    I say how much data is too little, and what to do about it.
+
+    Dropping is not symmetric. Removing a rater can push a subject below its
+    threshold, and removing a subject can push a rater below its own, so the
+    order changes the answer. I make that order a declared choice rather than
+    an accident of implementation.
+
+    My minimums may be raised but never lowered: the floors in ``contracts``
+    are what makes a row or column able to participate at all, and below them a
+    subject has nobody to disagree with. Raising a minimum is a study saying it
+    wants more evidence than the arithmetic strictly needs.
+
+    Attributes
+    ----------
+    min_ratings_per_rater:
+        How many subjects a rater must have rated to be kept.
+    min_ratings_per_subject:
+        How many raters must have rated a subject for it to be kept.
+    method:
+        ``raters_first`` drops raters, then subjects, once. ``subjects_first``
+        reverses that. ``iterate`` repeats until a pass drops nothing, which
+        can cascade a study down to very little -- the report shows each pass
+        so that is visible as a sequence rather than a single surprising total.
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    min_ratings_per_rater:   int
+    min_ratings_per_subject: int
+    method:                  str
+
+    _METHODS = ("raters_first", "subjects_first", "iterate")
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        min_ratings_per_rater:   int = MIN_RATINGS_PER_RATER,
+        min_ratings_per_subject: int = MIN_RATINGS_PER_SUBJECT,
+        method:                  str = "raters_first",
+    ) -> None:
+        self.min_ratings_per_rater   = min_ratings_per_rater
+        self.min_ratings_per_subject = min_ratings_per_subject
+        self.method                  = method
+        self._validate()
+
+    def _validate(self) -> None:
+        name = type(self).__name__
+
+        for field, value, floor in (
+            ("min_ratings_per_rater", self.min_ratings_per_rater,
+             MIN_RATINGS_PER_RATER),
+            ("min_ratings_per_subject", self.min_ratings_per_subject,
+             MIN_RATINGS_PER_SUBJECT),
+        ):
+            if not isinstance(value, int) or isinstance(value, bool):
+                raise TypeError(
+                    f"{name}.{field} must be an integer, got "
+                    f"{type(value).__name__}."
+                )
+            if value < floor:
+                raise ValueError(
+                    f"{name}.{field} is {value}, below the floor of {floor}. "
+                    f"That floor is what makes a rater or subject able to take "
+                    f"part in any comparison at all; it can be raised as a "
+                    f"study choice, never lowered."
+                )
+
+        if self.method not in self._METHODS:
+            raise ValueError(
+                f"{name}.method must be one of {list(self._METHODS)}, got "
+                f"{self.method!r}. Dropping is not symmetric, so the order is "
+                f"a choice you have to make."
+            )
+
+    @classmethod
+    def from_dict(cls, spec: dict) -> InsufficientDataPolicy:
+        unknown = sorted(set(spec) - {
+            "min_ratings_per_rater", "min_ratings_per_subject", "method"
+        })
+        if unknown:
+            raise KeyError(
+                f"insufficient_data_drops has unknown key(s) {unknown}. Known: "
+                f"min_ratings_per_rater, min_ratings_per_subject, method."
+            )
+        return cls(**spec)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, InsufficientDataPolicy):
+            return NotImplemented
+        return (
+            self.min_ratings_per_rater == other.min_ratings_per_rater
+            and self.min_ratings_per_subject == other.min_ratings_per_subject
+            and self.method == other.method
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        return (
+            f"InsufficientDataPolicy(per_rater="
+            f"{self.min_ratings_per_rater}, per_subject="
+            f"{self.min_ratings_per_subject}, method={self.method!r})"
+        )

interrater/cleaners/matching_report.py

@@ -0,0 +1,146 @@
+"""What happened to the strings, before anything became a number."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class MatchingReport:
+    """
+    I count what every cell turned into while it was still a string.
+
+    I am the first half of a cleaning report, covering everything that happens
+    before a value becomes a category position: cells that arrived null, cells
+    whose text meant nothing, cells that were repaired into a category, and
+    cells that were none of those and had to be dealt with by policy.
+
+    I count per rater as well as in total, because the totals hide the thing
+    most worth seeing. Four hundred missing cells spread evenly across four
+    reviewers is a study design; four hundred concentrated in one reviewer is
+    somebody who stopped halfway, and only the breakdown tells them apart.
+
+    For subjects I keep a distribution rather than a row per subject. With
+    sixteen hundred abstracts a per-subject listing is unreadable, and the
+    shape is the diagnostic anyway: a long tail of subjects each missing one
+    rating means something different from a handful missing all of them.
+
+    Attributes
+    ----------
+    n_cells:
+        Rater cells examined, excluding any subject id column.
+    n_null:
+        Cells that arrived null. Always missing, whatever the config says --
+        an empty spreadsheet cell is not a string and could not be listed.
+    missing_matched:
+        Per entry in ``missing_values``, how many cells it caught. Entries that
+        caught nothing are still listed: a declared marker that never fired is
+        worth seeing.
+    repairs:
+        Per entry in ``value_map``, how many cells it fixed. Empty when no
+        repairs were configured, and then omitted from the rendering entirely
+        -- somebody who declared no repairs is not asking about them.
+    unknown_values:
+        Strings that were neither missing nor a category, with how often each
+        appeared. Non-empty only when ``unknown_policy`` is ``empty``; under
+        ``error`` the clean stopped and there is no report to read.
+    per_rater_null, per_rater_missing, per_rater_unknown:
+        The same counts, keyed by rater name.
+    subject_missing_distribution:
+        How many subjects lost how many ratings: ``{0: 1204, 1: 380, 2: 33}``.
+    """
+
+    n_cells:         int
+    n_null:          int
+    missing_matched: dict[str, int] = field(default_factory=dict)
+    repairs:         dict[str, int] = field(default_factory=dict)
+    unknown_values:  dict[str, int] = field(default_factory=dict)
+
+    per_rater_null:    dict[str, int] = field(default_factory=dict)
+    per_rater_missing: dict[str, int] = field(default_factory=dict)
+    per_rater_unknown: dict[str, int] = field(default_factory=dict)
+
+    subject_missing_distribution: dict[int, int] = field(default_factory=dict)
+
+    @property
+    def n_missing(self) -> int:
+        """Cells with no rating: nulls plus every string that meant nothing."""
+        return self.n_null + sum(self.missing_matched.values())
+
+    @property
+    def n_repaired(self) -> int:
+        return sum(self.repairs.values())
+
+    @property
+    def n_unknown(self) -> int:
+        return sum(self.unknown_values.values())
+
+    def summary(self) -> dict:
+        """Plain types only, so this can be written beside the config."""
+        return {
+            "n_cells":         self.n_cells,
+            "n_null":          self.n_null,
+            "n_missing":       self.n_missing,
+            "n_repaired":      self.n_repaired,
+            "n_unknown":       self.n_unknown,
+            "missing_matched": dict(self.missing_matched),
+            "repairs":         dict(self.repairs),
+            "unknown_values":  dict(self.unknown_values),
+            "per_rater_null":    dict(self.per_rater_null),
+            "per_rater_missing": dict(self.per_rater_missing),
+            "per_rater_unknown": dict(self.per_rater_unknown),
+            "subject_missing_distribution": {
+                int(k): int(v) for k, v in self.subject_missing_distribution.items()
+            },
+        }
+
+    def render(self, verbosity: str = "summary") -> str:
+        pad   = "    "
+        fill  = self.n_missing / self.n_cells if self.n_cells else 0.0
+        lines = [
+            "Matching",
+            f"{pad}cells      : {self.n_cells:,}",
+            f"{pad}no rating  : {self.n_missing:,} ({fill:.1%}), "
+            f"of which {self.n_null:,} arrived null",
+        ]
+
+        if self.missing_matched:
+            shown = ", ".join(
+                f"{k!r} {v:,}" for k, v in self.missing_matched.items()
+            )
+            lines.append(f"{pad}matched    : {shown}")
+
+        # Nothing configured means nothing to report -- not "0 repairs".
+        if self.repairs:
+            shown = ", ".join(f"{k!r} {v:,}" for k, v in self.repairs.items())
+            lines.append(f"{pad}repaired   : {self.n_repaired:,} ({shown})")
+
+        if self.unknown_values:
+            top = sorted(
+                self.unknown_values.items(), key=lambda kv: -kv[1]
+            )[:3]
+            shown = ", ".join(f"{k!r} {v:,}" for k, v in top)
+            more  = len(self.unknown_values) - len(top)
+            lines.append(
+                f"{pad}unknown    : {self.n_unknown:,} treated as empty "
+                f"({shown}{f', +{more} more' if more > 0 else ''})"
+            )
+
+        if verbosity == "full":
+            for rater, n in self.per_rater_missing.items():
+                extra = self.per_rater_unknown.get(rater, 0)
+                lines.append(
+                    f"{pad}  {rater}: {n:,} without a rating"
+                    + (f", {extra:,} unknown" if extra else "")
+                )
+            if self.subject_missing_distribution:
+                shown = ", ".join(
+                    f"{n} subject(s) lost {k}"
+                    for k, n in sorted(self.subject_missing_distribution.items())
+                )
+                lines.append(f"{pad}  by subject: {shown}")
+
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:
+        return self.render()

interrater/contracts.py

@@ -0,0 +1,120 @@
+"""
+The array contract.
+
+Every workhorse function in this package operates on one array shape, described
+here. Nothing else in the package may invent a second convention.
+
+    category_index : np.ndarray, (n_subjects, n_raters), int8, C-contiguous
+
+Each cell is either a position in ``Categories.names`` or ``EMPTY``.
+
+Why subjects are rows
+---------------------
+Resampling subjects is then ``category_index[idx]`` -- a contiguous row gather,
+the fastest operation numpy offers. "Resample subjects" and "resample rows"
+become the same instruction, so a bootstrapper never rebuilds an object.
+
+Why int8 and not strings
+------------------------
+Integer positions let counts be a single ``bincount`` over a flattened index.
+At int8, a 1,617 x 4 study is 6.5 KB, so thousands of draws stay in cache.
+Names are resolved once, by the cleaner, and never enter the hot path.
+
+Why -1 and not NaN
+------------------
+NaN forces float64, forbids integer indexing, and turns every count into a
+masked operation. ``EMPTY`` keeps the array integral, and ``>= 0`` is a cheap
+validity mask.
+
+Where K comes from
+------------------
+K is ``Categories.n_categories`` and nothing else. It is never read off the
+data: ``category_index.max() + 1`` is a lower bound on K and never K itself.
+
+That gap is not academic. A bootstrap draw can miss a rare category entirely,
+and a function inferring K from that draw would silently compute with a smaller
+K than the study had. Kappa would survive it, since a category with no mass
+contributes a zero term either way -- but Gwet's AC1 and Brennan-Prediger put
+the number of categories in the chance term directly, so their values would
+move draw to draw for no reason but the inference. The resulting interval would
+look like sampling variability and would not be.
+
+Only an analyser unpacks a Ratings. It reads K from the semantics and passes
+it down to the workhorse functions and bootstrappers, which take plain arrays.
+Keeping that unpacking in one layer is what stops K from ever being read off a
+draw.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+# ----------------------------------------------------------------- Sentinel #
+
+EMPTY: int = -1
+"""No rating exists in this cell.
+
+One sentinel, one meaning: this cell takes no part in any calculation, but it
+holds its place so pairwise overlaps and rater permutations stay correct.
+
+I carry no reason. A subject never shown to a rater and a subject shown and
+left blank are both simply empty here, because at this level the only fact that
+matters is that there is no rating. Why a cell is empty is the cleaner's
+business, is recorded in its report, and changes what you can say about a rater
+-- never what any coefficient computes.
+
+'Unsure', 'abstain', and anything else a study designer chose to offer are
+*categories*, not empties. They were answers. They live in ``Categories.names``
+and count like any other response.
+"""
+
+# ------------------------------------------------------------------- Dtypes #
+
+INDEX_DTYPE: np.dtype = np.dtype(np.int8)
+"""Storage type for the category index array. Caps a study at 127 categories."""
+
+LABEL_DTYPE: np.dtype = np.dtype(object)
+"""Storage type for subject ids.
+
+``object`` holds real Python strings, so an id of any length survives intact.
+A fixed-width dtype such as ``U32`` would be faster, but it truncates anything
+longer without complaint, and a silently shortened identifier is a data
+corruption nobody would notice.
+"""
+
+# ----------------------------------------------------------------- Minimums #
+
+MIN_RATERS: int = 2
+"""Fewer than two raters is not an agreement problem."""
+
+MIN_SUBJECTS: int = 2
+"""Fewer than two subjects leaves nothing to disagree about."""
+
+MIN_RATINGS_PER_RATER: int = 2
+"""Floor on how many subjects a rater must have rated.
+
+A rater with one rating joins one comparison and describes nobody. A cleaner
+may set its threshold higher as a study choice, never lower.
+"""
+
+MIN_RATINGS_PER_SUBJECT: int = 2
+"""Floor on how many raters must have rated a subject.
+
+A subject rated once has nobody to disagree with: it joins no pairwise
+comparison and contributes nothing to any group coefficient. A cleaner may set
+its threshold higher as a study choice, never lower.
+"""
+
+MIN_OVERLAP_HARD: int = 2
+"""Mathematical floor on co-rated subjects for a pairwise coefficient.
+
+Below two, expected agreement cannot be estimated and kappa is undefined. This
+is a validity threshold, not a reliability one: a pair meeting it can still
+produce a number nobody should trust.
+
+The *practical* floor is higher and is not a constant. Under heavy skew,
+expected agreement is large and kappa has little room left to work in, so more
+co-rated subjects are needed for the same precision. That threshold depends on
+prevalence and is not yet in this package; see the roadmap entry on overlap
+adequacy.
+"""

interrater/data_objects/__init__.py

@@ -0,0 +1,3 @@
+from .ratings import Ratings
+
+__all__ = ["Ratings"]