interrater 0.0.1.dev0__py3-none-any.whl

interrater/__init__.py

@@ -0,0 +1 @@
+"""interrater -- agreement between raters, with auditable inputs."""

interrater/cleaners/__init__.py

@@ -0,0 +1,19 @@
+from .category_report import CategoryReport
+from .cleaner_config import CleanerConfig
+from .cleaning_report import CleaningReport
+from .drop_report import DropPass, DropReport
+from .frame_layout import FrameLayout
+from .insufficient_data_policy import MAX_DROP_PASSES, InsufficientDataPolicy
+from .matching_report import MatchingReport
+
+__all__ = [
+    "CleanerConfig",
+    "FrameLayout",
+    "InsufficientDataPolicy",
+    "MAX_DROP_PASSES",
+    "CleaningReport",
+    "MatchingReport",
+    "DropReport",
+    "DropPass",
+    "CategoryReport",
+]

interrater/cleaners/_string_utils.py

@@ -0,0 +1,37 @@
+"""
+String work for the cleaner.
+
+Free functions over plain strings. Nothing here knows what a study is, raises,
+or holds state. The config calls into it to check its own consistency before
+any data exists; the cleaner calls the same functions on every cell. One rule,
+one implementation -- a config validated under one interpretation and matched
+under another would be worse than no validation at all.
+"""
+
+from __future__ import annotations
+
+
+def fold(value: str, *, case_sensitive: bool, strip_whitespace: bool) -> str:
+    """
+    Normalise a string the way it will be matched.
+
+    Under the defaults ``" NA "`` becomes ``"na"``, so a whitespace variant
+    need not appear in every list and a value cannot match a category
+    case-insensitively while missing the missing-values list.
+
+    Named for ``str.casefold``, which is the comparison-grade lowercase: it
+    handles cases ``lower()`` does not, such as German ``ß`` folding to ``ss``.
+    """
+    if strip_whitespace:
+        value = value.strip()
+    return value if case_sensitive else value.casefold()
+
+
+def fold_all(
+    values, *, case_sensitive: bool, strip_whitespace: bool
+) -> list[str]:
+    """``fold`` over an iterable, in order."""
+    return [
+        fold(v, case_sensitive=case_sensitive, strip_whitespace=strip_whitespace)
+        for v in values
+    ]

interrater/cleaners/category_report.py

@@ -0,0 +1,96 @@
+"""Which declared categories were actually used."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class CategoryReport:
+    """
+    I say which of the declared categories anybody chose.
+
+    A category nobody chose is not automatically wrong. If the scheme offered
+    five grades and no student earned a D, that is a fact about the cohort. But
+    it is also the shape a mistake takes when a study is not what its owner
+    thinks it is, which is why the default policy stops rather than shrugs.
+
+    Keeping an unused category is the safe answer, because K is a fact about
+    what raters could have said. Dropping one changes any coefficient whose
+    chance term counts categories -- Gwet's AC1 and Brennan-Prediger both --
+    while leaving kappa untouched, so the damage is quiet and easy to miss.
+
+    I also record the rare categories, which the unused check does not catch
+    and which are more dangerous than the empty ones. A category chosen once in
+    three hundred cells passes any test for being used, and every statistic
+    involving it is noise.
+
+    Attributes
+    ----------
+    counts:
+        Category name to how many ratings it received.
+    unused:
+        Categories with no ratings at all.
+    policy:
+        What ``unused_categories`` was set to.
+    action:
+        What that policy did. Under ``error`` there is no report to read, so in
+        practice this is 'kept' or 'dropped'.
+    rare_threshold:
+        Below this many ratings a category is called rare.
+    """
+
+    counts:         dict[str, int]
+    policy:         str
+    action:         str = "kept"
+    rare_threshold: int = 10
+    unused:         tuple[str, ...] = ()
+
+    @property
+    def rare(self) -> dict[str, int]:
+        """Used, but so seldom that anything involving them is noise."""
+        return {
+            name: n
+            for name, n in self.counts.items()
+            if 0 < n < self.rare_threshold
+        }
+
+    def summary(self) -> dict:
+        return {
+            "counts":         dict(self.counts),
+            "unused":         list(self.unused),
+            "rare":           dict(self.rare),
+            "rare_threshold": self.rare_threshold,
+            "policy":         self.policy,
+            "action":         self.action,
+        }
+
+    def render(self, verbosity: str = "summary") -> str:
+        pad   = "    "
+        total = sum(self.counts.values()) or 1
+        lines = ["Categories"]
+
+        if verbosity == "full":
+            for name, n in self.counts.items():
+                lines.append(f"{pad}{name:<14} {n:>8,}  ({n / total:.1%})")
+        else:
+            shown = ", ".join(
+                f"{name} {n / total:.0%}" for name, n in self.counts.items()
+            )
+            lines.append(f"{pad}used       : {shown}")
+
+        if self.unused:
+            lines.append(
+                f"{pad}unused     : {list(self.unused)} ({self.action})"
+            )
+        if self.rare:
+            shown = ", ".join(f"{k} ({v})" for k, v in self.rare.items())
+            lines.append(
+                f"{pad}rare       : {shown} -- fewer than "
+                f"{self.rare_threshold} ratings; statistics involving these "
+                f"are not estimable"
+            )
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:
+        return self.render()

interrater/cleaners/cleaner_config.py

@@ -0,0 +1,308 @@
+"""Every cleaning decision, declared in one place and versioned with the study."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ._string_utils import fold
+from .frame_layout import FrameLayout
+from .insufficient_data_policy import InsufficientDataPolicy
+
+
+class CleanerConfig:
+    """
+    I am every decision a cleaner makes, written down before it runs.
+
+    I am the methods section of a paper in machine-readable form. Which strings
+    meant nothing, which were repaired and into what, what happened to a
+    response nobody declared, how thin the data was allowed to get -- all of it
+    stated in one file that is versioned with the analysis rather than
+    reconstructed from memory later.
+
+    I know nothing about any particular study. I have never seen a category
+    list or a rater name, which is why I can only check my own shape: that my
+    keys are strings, that a value is not claimed twice, that a minimum is not
+    below its floor. Whether ``value_map`` sends a string to a category that
+    actually exists is a question about a study, and the cleaner -- which holds
+    both me and the semantics -- is the one that can ask it and give a useful
+    answer. That check runs before any row is touched.
+
+    I delegate two concerns that have rules of their own. ``FrameLayout``
+    describes the file rather than the study, and is the part of me that does
+    not travel to a second export. ``InsufficientDataPolicy`` owns dropping,
+    which needs an order and a loop and so is more than a pair of numbers. What
+    is left on me directly is string matching, which is the one thing every
+    other part depends on being consistent.
+
+    Attributes
+    ----------
+    frame_layout:
+        How to read the frame. See ``FrameLayout``.
+    case_sensitive:
+        Whether string matching distinguishes case. Governs everything matched:
+        category names, ``missing_values``, and ``value_map`` keys alike, so
+        there is no way for a value to match one and miss another.
+    strip_whitespace:
+        Whether to trim before matching. On, so ``" NA "`` catches without
+        needing a whitespace variant in every list.
+    missing_values:
+        Strings that mean no rating. A separate list rather than entries in
+        ``value_map`` because absence is the decision most worth seeing at a
+        glance. Nulls are always missing regardless of what is listed here --
+        an empty spreadsheet cell arrives as a null, not as a string, so it
+        could not be listed anyway.
+    value_map:
+        Repairs: a string as it appears in the file, mapped to the category it
+        was meant to be.
+    unknown_policy:
+        What to do with a value that survives normalisation and is still not a
+        category. ``error`` by default, because a response nobody declared is
+        usually a sign the data is not what its owner thinks it is.
+    unused_categories:
+        What to do when a declared category was never chosen. ``error`` by
+        default, for the same reason. ``keep`` is the safe alternative: K is a
+        fact about what raters could have said, and dropping a category would
+        move any coefficient whose chance term counts them.
+    insufficient_data_drops:
+        How thin the data may get. See ``InsufficientDataPolicy``.
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    frame_layout:            FrameLayout
+    case_sensitive:          bool
+    strip_whitespace:        bool
+    missing_values:          tuple[str, ...]
+    value_map:               dict[str, str]
+    unknown_policy:          str
+    unused_categories:       str
+    insufficient_data_drops: InsufficientDataPolicy
+
+    _UNKNOWN_POLICIES = ("error", "empty")
+    _UNUSED_POLICIES  = ("error", "keep", "drop")
+
+    DEFAULT_MISSING_VALUES = ("", "NA", "N/A")
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        frame_layout:            FrameLayout | None = None,
+        case_sensitive:          bool = False,
+        strip_whitespace:        bool = True,
+        missing_values:          object = None,
+        value_map:               dict[str, str] | None = None,
+        unknown_policy:          str = "error",
+        unused_categories:       str = "error",
+        insufficient_data_drops: InsufficientDataPolicy | None = None,
+    ) -> None:
+        self.frame_layout      = frame_layout or FrameLayout()
+        self.case_sensitive    = case_sensitive
+        self.strip_whitespace  = strip_whitespace
+        self.missing_values    = tuple(
+            self.DEFAULT_MISSING_VALUES if missing_values is None
+            else missing_values
+        )
+        self.value_map         = dict(value_map or {})
+        self.unknown_policy    = unknown_policy
+        self.unused_categories = unused_categories
+        self.insufficient_data_drops = (
+            insufficient_data_drops or InsufficientDataPolicy()
+        )
+        self._validate()
+
+    # ---- Validation: shape only. I have never seen a study. ---- #
+
+    def _validate(self) -> None:
+        self._validate_delegates()
+        self._validate_flags()
+        self._validate_missing_values()
+        self._validate_value_map()
+        self._validate_no_double_claim()
+        self._validate_policies()
+
+    def _validate_delegates(self) -> None:
+        if not isinstance(self.frame_layout, FrameLayout):
+            raise TypeError(
+                f"CleanerConfig.frame_layout must be a FrameLayout, got "
+                f"{type(self.frame_layout).__name__}."
+            )
+        if not isinstance(self.insufficient_data_drops, InsufficientDataPolicy):
+            raise TypeError(
+                f"CleanerConfig.insufficient_data_drops must be an "
+                f"InsufficientDataPolicy, got "
+                f"{type(self.insufficient_data_drops).__name__}."
+            )
+
+    def _validate_flags(self) -> None:
+        for field in ("case_sensitive", "strip_whitespace"):
+            if not isinstance(getattr(self, field), bool):
+                raise TypeError(
+                    f"CleanerConfig.{field} must be a boolean, got "
+                    f"{type(getattr(self, field)).__name__}."
+                )
+
+    def _validate_missing_values(self) -> None:
+        bad = [v for v in self.missing_values if not isinstance(v, str)]
+        if bad:
+            raise TypeError(
+                f"CleanerConfig.missing_values must hold strings, got "
+                f"{[type(v).__name__ for v in bad[:3]]}. A null cell is always "
+                f"missing and cannot be listed here."
+            )
+
+    def _validate_value_map(self) -> None:
+        if not isinstance(self.value_map, dict):
+            raise TypeError(
+                f"CleanerConfig.value_map must be a mapping of file string to "
+                f"category name, got {type(self.value_map).__name__}."
+            )
+        for key, value in self.value_map.items():
+            if not isinstance(key, str) or not isinstance(value, str):
+                raise TypeError(
+                    f"CleanerConfig.value_map must map string to string, but "
+                    f"{key!r} -> {value!r} is "
+                    f"{type(key).__name__} -> {type(value).__name__}. To mark "
+                    f"a string as absent, list it in missing_values instead."
+                )
+            if not key:
+                raise ValueError(
+                    "CleanerConfig.value_map has an empty key. An empty cell "
+                    "is handled by missing_values, not here."
+                )
+
+    def _validate_no_double_claim(self) -> None:
+        """
+        A string cannot both mean nothing and mean something.
+
+        Folded before comparing, using the same rule the cleaner will apply to
+        data, so a config that validates here cannot behave differently there.
+        """
+        missing = {self._fold(v) for v in self.missing_values}
+        claimed = sorted(
+            key for key in self.value_map if self._fold(key) in missing
+        )
+        if claimed:
+            raise ValueError(
+                f"CleanerConfig claims {len(claimed)} string(s) twice "
+                f"(e.g. {claimed[:3]}): they appear in missing_values and are "
+                f"also mapped by value_map. Decide whether each means no "
+                f"rating or a category, and list it once."
+                + ("" if self.case_sensitive else
+                   " Matching is case-insensitive, so entries differing only "
+                   "in case collide.")
+            )
+
+    def _validate_policies(self) -> None:
+        if self.unknown_policy not in self._UNKNOWN_POLICIES:
+            raise ValueError(
+                f"CleanerConfig.unknown_policy must be one of "
+                f"{list(self._UNKNOWN_POLICIES)}, got {self.unknown_policy!r}. "
+                f"'error' stops on a response nobody declared; 'empty' treats "
+                f"it as no rating."
+            )
+        if self.unused_categories not in self._UNUSED_POLICIES:
+            raise ValueError(
+                f"CleanerConfig.unused_categories must be one of "
+                f"{list(self._UNUSED_POLICIES)}, got "
+                f"{self.unused_categories!r}."
+            )
+
+    def _fold(self, value: str) -> str:
+        """My matching rule, applied to one string."""
+        return fold(
+            value,
+            case_sensitive   = self.case_sensitive,
+            strip_whitespace = self.strip_whitespace,
+        )
+
+    # --------------------------------------------------------- Declarative #
+
+    @classmethod
+    def from_dict(cls, spec: dict) -> CleanerConfig:
+        spec   = dict(spec)
+        layout = spec.pop("frame_layout", None)
+        drops  = spec.pop("insufficient_data_drops", None)
+        known  = {
+            "case_sensitive", "strip_whitespace", "missing_values",
+            "value_map", "unknown_policy", "unused_categories",
+        }
+        unknown = sorted(set(spec) - known)
+        if unknown:
+            raise KeyError(
+                f"CleanerConfig has unknown key(s) {unknown}. Known: "
+                f"{sorted(known)}, frame_layout, insufficient_data_drops."
+            )
+        return cls(
+            frame_layout=FrameLayout.from_dict(layout) if layout else None,
+            insufficient_data_drops=(
+                InsufficientDataPolicy.from_dict(drops) if drops else None
+            ),
+            **spec,
+        )
+
+    @classmethod
+    def build_from_yaml(cls, path: str | Path) -> CleanerConfig:
+        """
+        Build from a YAML file:
+
+        .. code-block:: yaml
+
+            frame_layout:
+              use_column_names: true
+              subject_id_col: abstract_id
+            case_sensitive: false
+            missing_values: ["", "NA", "NOT SHOWN"]
+            value_map:
+              inc: include
+              exc: exclude
+            unknown_policy: error
+            unused_categories: error
+            insufficient_data_drops:
+              min_ratings_per_rater: 2
+              min_ratings_per_subject: 2
+              method: raters_first
+        """
+        import yaml
+
+        with open(path, "r", encoding="utf-8") as handle:
+            spec = yaml.safe_load(handle)
+        if not isinstance(spec, dict):
+            raise TypeError(
+                f"{path} must contain a YAML mapping, got "
+                f"{type(spec).__name__}."
+            )
+        return cls.from_dict(spec)
+
+    # ------------------------------------------------------------ Public API #
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, CleanerConfig):
+            return NotImplemented
+        return all(
+            getattr(self, field) == getattr(other, field)
+            for field in (
+                "frame_layout", "case_sensitive", "strip_whitespace",
+                "missing_values", "value_map", "unknown_policy",
+                "unused_categories", "insufficient_data_drops",
+            )
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        drops = self.insufficient_data_drops
+        return (
+            f"CleanerConfig(\n"
+            f"    frame_layout      = {self.frame_layout}\n"
+            f"    case_sensitive    = {self.case_sensitive}\n"
+            f"    strip_whitespace  = {self.strip_whitespace}\n"
+            f"    missing_values    = {list(self.missing_values)}\n"
+            f"    value_map         = {len(self.value_map)} repair(s)\n"
+            f"    unknown_policy    = {self.unknown_policy!r}\n"
+            f"    unused_categories = {self.unused_categories!r}\n"
+            f"    drops             = per_rater>={drops.min_ratings_per_rater}, "
+            f"per_subject>={drops.min_ratings_per_subject}, "
+            f"{drops.method!r}\n"
+            f")"
+        )

interrater/cleaners/cleaning_report.py

@@ -0,0 +1,125 @@
+"""What a cleaner did, in a form you can read or file away."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .category_report import CategoryReport
+from .cleaner_config import CleanerConfig
+from .drop_report import DropReport
+from .matching_report import MatchingReport
+
+_VERBOSITIES = ("summary", "full")
+
+
+@dataclass(frozen=True)
+class CleaningReport:
+    """
+    I am the receipt for a clean. The soup does not remember the carrots; I do.
+
+    A ``Ratings`` describes its own contents and nothing about where they came
+    from, which is what keeps it comparable to any other ratings with the same
+    values. Everything that was normalised, repaired, treated as absent or
+    dropped along the way lives in me instead, beside the config that decided
+    it -- so a methods section can be read off rather than reconstructed.
+
+    I always hold everything. Verbosity is a property of how I am printed,
+    never of what I computed: two runs of the same data must produce the same
+    record, or a reader cannot tell whether a section is missing because
+    nothing happened or because somebody turned it off. Counting is cheap next
+    to the work of matching every cell, so there is nothing to buy back by
+    computing less.
+
+    I am frozen. I describe a run that already finished.
+
+    Attributes
+    ----------
+    config:
+        The rules that were applied. Without it my numbers say what happened
+        but not why, and the two together are the whole account.
+    shape_in:
+        Rows and columns of the frame handed to the cleaner.
+    shape_out:
+        Subjects and raters of the Ratings that came out.
+    matching, drops, categories:
+        The three stages. See ``MatchingReport``, ``DropReport``,
+        ``CategoryReport``.
+    """
+
+    config:     CleanerConfig
+    shape_in:   tuple[int, int]
+    shape_out:  tuple[int, int]
+    matching:   MatchingReport
+    drops:      DropReport
+    categories: CategoryReport
+
+    # ------------------------------------------------------------- Reading #
+
+    @property
+    def n_subjects_lost(self) -> int:
+        return self.shape_in[0] - self.shape_out[0]
+
+    @property
+    def n_raters_lost(self) -> int:
+        return self.drops.n_raters_dropped
+
+    @property
+    def survived(self) -> float:
+        """Fraction of input rows that made it into the Ratings."""
+        return self.shape_out[0] / self.shape_in[0] if self.shape_in[0] else 0.0
+
+    def summary(self) -> dict:
+        """
+        Everything I hold, in plain types.
+
+        Complete at every verbosity: this is the machine-readable record, and
+        it would be useless if a display setting could change it. Write it
+        beside the config and the pair reproduces the run.
+        """
+        return {
+            "shape_in":  list(self.shape_in),
+            "shape_out": list(self.shape_out),
+            "n_subjects_lost": self.n_subjects_lost,
+            "n_raters_lost":   self.n_raters_lost,
+            "survived":        round(self.survived, 6),
+            "matching":   self.matching.summary(),
+            "drops":      self.drops.summary(),
+            "categories": self.categories.summary(),
+        }
+
+    def to_yaml(self, path: str | Path) -> None:
+        """Write my summary beside the config that produced it."""
+        import yaml
+
+        with open(path, "w", encoding="utf-8") as handle:
+            yaml.safe_dump(self.summary(), handle, sort_keys=False)
+
+    # ------------------------------------------------------------ Printing #
+
+    def render(self, verbosity: str = "summary") -> str:
+        if verbosity not in _VERBOSITIES:
+            raise ValueError(
+                f"verbosity must be one of {list(_VERBOSITIES)}, got "
+                f"{verbosity!r}."
+            )
+        rows_in, cols_in   = self.shape_in
+        rows_out, cols_out = self.shape_out
+        head = [
+            "CleaningReport",
+            f"    in         : {rows_in:,} row(s) x {cols_in} column(s)",
+            f"    out        : {rows_out:,} subject(s) x {cols_out} rater(s) "
+            f"({self.survived:.1%} of rows kept)",
+            "",
+        ]
+        body = [
+            self.matching.render(verbosity),
+            "",
+            self.drops.render(verbosity),
+            "",
+            self.categories.render(verbosity),
+        ]
+        return "\n".join(head + body)
+
+    def __repr__(self) -> str:
+        return self.render()