interrater 0.0.1.dev0__py3-none-any.whl

interrater/semantics/ratings_semantics.py

@@ -0,0 +1,253 @@
+"""What a ratings array means, declared before the data is seen."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+
+from ..contracts import MIN_RATERS, MIN_SUBJECTS
+from .categories import Categories, build_categories
+
+
+class RatingsSemantics:
+    """
+    I say what a ratings array means. A numpy array does not know what it is.
+
+    My biggest contribution is that I can be built from a YAML file: one
+    declaration that constructs me, is versioned with the analysis, and doubles
+    as the documentation of what the study was.
+
+    I name the columns, I own the categories that give the numbers meaning, and
+    I record what the data was expected to look like. I hold no ratings and do
+    no counting.
+
+    I am written before the data is read, not derived from it. That is the
+    point of me: if a spreadsheet arrives with five rater columns when four
+    reviewers were hired, the fifth is a mistake to be reported, not a rater to
+    be invented. Declaring the study up front is what turns a surprise into an
+    error message.
+
+    I am reusable. The same four reviewers screening a second review need the
+    same me, which is why I know nothing about any particular dataset -- no
+    subject ids, no counts, nothing a cleaner discovered. My ``n_subjects`` is
+    an expectation you may state and I may check, never a field anything fills
+    in later.
+
+    Attributes
+    ----------
+    raters:
+        Column labels, in column order, never sorted. Given a count instead of
+        names I generate ``rater0, rater1, ...``, so a label is file-relative:
+        ``rater0`` means "the first rater column of whatever was fed in", not a
+        person who carries across studies. Numbering starts at zero so a label
+        matches its column index.
+    categories:
+        What the integers in the array mean, and how far apart they are. See
+        ``Categories``.
+    n_subjects:
+        How many rows the data should have, or None for no expectation. Stated,
+        it is checked; absent, nothing is checked. Never populated after the
+        fact -- that would tie me to one dataset and end my reuse.
+    unique_subjects:
+        Whether repeated subject ids are an error. True by default, because a
+        repeated id can be frequently due to an export bug. Set False for a study that
+        deliberately re-shows subjects to measure intra-rater consistency.
+        Bootstrap draws repeat rows too, but they bypass validation entirely
+        and never consult me.
+
+    Examples
+    --------
+    >>> from interrater.semantics import UnorderedCategories
+    >>> sem = RatingsSemantics(
+    ...     raters     = ["ann", "bob", "cara", "dev"],
+    ...     categories = UnorderedCategories(("include", "unsure", "exclude")),
+    ... )
+    >>> sem.n_raters
+    4
+    >>> RatingsSemantics(raters=3, categories=sem.categories).raters
+    ('rater0', 'rater1', 'rater2')
+    """
+
+    # ----------------------------------------------------------- Attributes #
+
+    raters:          tuple[str, ...]
+    # Always a tuple, even when a count was passed: __init__ generates the
+    # names immediately, so nothing downstream ever sees an int.
+    categories:      Categories
+    n_subjects:      int | None
+    unique_subjects: bool
+
+    # --------------------------------------------------------- Construction #
+
+    def __init__(
+        self,
+        raters:          Sequence[str] | int,
+        categories:      Categories,
+        n_subjects:      int | None = None,
+        unique_subjects: bool = True,
+    ) -> None:
+        self.raters          = self._as_rater_names(raters)
+        self.categories      = categories
+        self.n_subjects      = n_subjects
+        self.unique_subjects = unique_subjects
+        self._validate()
+
+    @staticmethod
+    def _as_rater_names(raters: Sequence[str] | int) -> tuple[str, ...]:
+        """A count becomes generated names; names are taken as given."""
+        if isinstance(raters, int):
+            if raters < MIN_RATERS:
+                raise ValueError(
+                    f"RatingsSemantics needs at least {MIN_RATERS} raters, "
+                    f"got {raters}. Agreement is undefined for a single rater."
+                )
+            return tuple(f"rater{i}" for i in range(raters))
+        return tuple(str(name) for name in raters)
+
+    def _validate(self) -> None:
+        name = type(self).__name__
+
+        if not isinstance(self.categories, Categories):
+            raise TypeError(
+                f"{name}.categories must be a Categories, got "
+                f"{type(self.categories).__name__}. Use UnorderedCategories "
+                f"for a nominal scheme, OrderedCategories for a ranked one, or "
+                f"WeightedCategories to supply distances directly."
+            )
+
+        if len(self.raters) < MIN_RATERS:
+            raise ValueError(
+                f"{name} needs at least {MIN_RATERS} raters, got "
+                f"{len(self.raters)}. Agreement is undefined for a single "
+                f"rater."
+            )
+
+        seen: set[str] = set()
+        dupes = [r for r in self.raters if r in seen or seen.add(r)]
+        if dupes:
+            raise ValueError(
+                f"{name}.raters contains {len(dupes)} duplicate name(s) "
+                f"(e.g. {dupes[:3]}). Two columns naming the same rater would "
+                f"be counted as independent by every coefficient."
+            )
+
+        blank = [i for i, r in enumerate(self.raters) if not r.strip()]
+        if blank:
+            raise ValueError(
+                f"{name}.raters has {len(blank)} blank name(s) at position(s) "
+                f"{blank[:3]}. Give every column a name, or pass a count and "
+                f"let me generate rater0, rater1, ..."
+            )
+
+        if self.n_subjects is not None and self.n_subjects < MIN_SUBJECTS:
+            raise ValueError(
+                f"{name}.n_subjects must be at least {MIN_SUBJECTS} if "
+                f"stated, got {self.n_subjects}. Pass None for no expectation."
+            )
+
+    # -------------------------------------------------------- Declarative #
+
+    @classmethod
+    def from_dict(cls, spec: dict) -> RatingsSemantics:
+        """
+        Build from a plain mapping, as parsed from YAML.
+
+        ``raters`` is either a list of names or an integer count.
+        ``categories`` is a mapping dispatched by ``kind``; see
+        ``build_categories``.
+        """
+        missing = {"raters", "categories"} - set(spec)
+        if missing:
+            raise KeyError(
+                f"RatingsSemantics config is missing {sorted(missing)}. A "
+                f"study must declare who rated and what they could say."
+            )
+        return cls(
+            raters          = spec["raters"],
+            categories      = build_categories(spec["categories"]),
+            n_subjects      = spec.get("n_subjects"),
+            unique_subjects = spec.get("unique_subjects", True),
+        )
+
+    @classmethod
+    def build_from_yaml(cls, path: str | Path) -> RatingsSemantics:
+        """
+        Build from a YAML file.
+
+        The file is the study declaration, versioned alongside the analysis:
+
+        .. code-block:: yaml
+
+            raters: [ann, bob, cara, dev]
+            categories:
+              kind: ordered
+              names: [mild, moderate, severe]
+              weighting: quadratic
+            n_subjects: 1617
+            unique_subjects: true
+        """
+        import yaml
+
+        with open(path, "r", encoding="utf-8") as handle:
+            spec = yaml.safe_load(handle)
+        if not isinstance(spec, dict):
+            raise TypeError(
+                f"{path} must contain a YAML mapping, got "
+                f"{type(spec).__name__}."
+            )
+        return cls.from_dict(spec)
+
+    # ------------------------------------------------------------ Public API #
+
+    @property
+    def n_raters(self) -> int:
+        return len(self.raters)
+
+    @property
+    def n_categories(self) -> int:
+        """K, owned by my categories. Never inferred from any array."""
+        return self.categories.n_categories
+
+    def index_of_rater(self, rater: str) -> int:
+        """Which column holds ``rater``."""
+        try:
+            return self.raters.index(rater)
+        except ValueError:
+            raise KeyError(
+                f"{type(self).__name__} has no rater {rater!r}. Known: "
+                f"{list(self.raters)}."
+            ) from None
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Two of me are equal when we declare the same study.
+
+        ``other`` is annotated ``object`` and returns ``NotImplemented`` rather
+        than raising, because that is the protocol: Python then tries the
+        reflected comparison and falls back to False. Raising here would break
+        ``sem in [a, b]``, dict comparison, and any assertion that compares me
+        with something else -- all of which are supposed to answer False, not
+        explode. A test helper is the place for a loud, diffing comparison.
+        """
+        if not isinstance(other, RatingsSemantics):
+            return NotImplemented
+        return (
+            self.raters == other.raters
+            and self.categories == other.categories
+            and self.n_subjects == other.n_subjects
+            and self.unique_subjects == other.unique_subjects
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        expected = "any" if self.n_subjects is None else f"{self.n_subjects:,}"
+        return (
+            f"RatingsSemantics(\n"
+            f"    raters          = {list(self.raters)}\n"
+            f"    categories      = {type(self.categories).__name__}"
+            f"{list(self.categories.names)}\n"
+            f"    n_subjects      = {expected}\n"
+            f"    unique_subjects = {self.unique_subjects}\n"
+            f")"
+        )

interrater-0.0.1.dev0.dist-info/METADATA

@@ -0,0 +1,82 @@
+Metadata-Version: 2.5
+Name: interrater
+Version: 0.0.1.dev0
+Summary: Inter-rater agreement for systematic reviews: auditable cleaning, bootstrap intervals, and simulated ratings with known truth
+Project-URL: Homepage, https://github.com/djarenas/Inter-Rater
+Project-URL: Repository, https://github.com/djarenas/Inter-Rater
+Project-URL: Issues, https://github.com/djarenas/Inter-Rater/issues
+Project-URL: Prior version (arXiv), https://arxiv.org/abs/1809.05731
+Project-URL: Prior version (Zenodo), https://doi.org/10.5281/zenodo.1227660
+Author: Daniel J. Arenas
+License-Expression: GPL-3.0-or-later
+License-File: LICENSE
+Keywords: bootstrap,cohens-kappa,fleiss-kappa,inter-rater-reliability,interrater-agreement,krippendorff-alpha,systematic-review
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# interrater
+
+Inter-rater agreement for systematic reviews and other studies where not every
+rater sees every subject.
+
+**This release reserves the package name. It is not yet usable.** Version 2 is
+under active development; follow
+[the repository](https://github.com/djarenas/Inter-Rater) for progress.
+
+## Background
+
+`interrater` is the successor to
+[Inter-Rater](https://github.com/djarenas/Inter-Rater), described in
+[arXiv:1809.05731](https://arxiv.org/abs/1809.05731) (2018) and archived at
+[Zenodo](https://doi.org/10.5281/zenodo.1227660). Version 1 has been used in
+published systematic reviews to compute Fleiss' kappa and pairwise Cohen's
+kappa across multiple reviewers.
+
+Version 2 is a complete rewrite with a different API. Version 1 remains
+available at git tag `v1.5`.
+
+## What version 2 adds
+
+- **Bootstrap confidence intervals.** Version 1 reported an analytic standard
+  error for the average of pairwise kappas, which treats those kappas as
+  independent. They are not: two pairwise coefficients sharing a rater are
+  correlated, and the resulting interval is too narrow. Resampling subjects
+  preserves that correlation without having to model it.
+
+- **An auditable cleaning layer.** Which strings meant nothing, which were
+  repaired and into what, how thin the data was allowed to get: declared in a
+  versioned config file and reported afterwards, rather than reconstructed from
+  memory when a reviewer asks.
+
+- **Simulated ratings with known truth.** Generate data from an explicit model
+  of rater accuracy, category prevalence and subject assignment, so an
+  estimator can be checked against an answer known in advance and a confidence
+  interval can be checked for actually covering it.
+
+- **Support for unbalanced designs.** Where each subject is rated by a subset
+  of the reviewer pool rather than all of them, which is what most reviews can
+  actually afford.
+
+## Requirements
+
+Python 3.10 or later.
+
+## License
+
+GNU General Public License v3 or later. Same license as version 1.

interrater-0.0.1.dev0.dist-info/RECORD

@@ -0,0 +1,21 @@
+interrater/__init__.py,sha256=PicnolOWJRpMczbpOTps44vc7o4SFLNODjd3IoP1IOg,69
+interrater/contracts.py,sha256=MiKUQLtAw60mAZo3FxaxPYveNW1eFvLUfLGHRnAYGV8,4846
+interrater/cleaners/__init__.py,sha256=ewgV1RO2se9wXbKXH63TMP1kaOBEiFy9TLbxqjJL8eE,544
+interrater/cleaners/_string_utils.py,sha256=YY72TLUVtE1sH_g6fKZPN9n5B74wVsBmWYg_AKuR1xI,1333
+interrater/cleaners/category_report.py,sha256=BkjvJt-qpoZTDQ3pogmsjQVSAv-5NUlqmOZWlBTtANA,3376
+interrater/cleaners/cleaner_config.py,sha256=bbcqUCuX88KdaMT63mdDVYyp6mdmEeeHu1Au5_yCREk,12866
+interrater/cleaners/cleaning_report.py,sha256=zGF1njuF28kBhfY_pLX_U-d_hlVXAMH8K1DCizQRkVE,4462
+interrater/cleaners/drop_report.py,sha256=lDmCc5DEELUqyD5EA8rrWyEvrWXynb0jRercc0MOVUw,5599
+interrater/cleaners/frame_layout.py,sha256=5p5F3ILUGmX_VFnsr54yz1bokpqJG7tAifeImmtiswU,3819
+interrater/cleaners/insufficient_data_policy.py,sha256=MlwS1iz54FUpXgb5L-Yf9mZnF8CQ_NTGx3KYbS_w1Zs,4852
+interrater/cleaners/matching_report.py,sha256=Hs8R0cH2zBgGzfdA598RJcDecYqkrX6EY7r1Yjf6i8U,6132
+interrater/data_objects/__init__.py,sha256=f0Wo0vs-xVBPE06cCCd6OpF6t1jb18oLw76n0IduCi8,52
+interrater/data_objects/_ratings_validation.py,sha256=IUgto3ltjHIw0i2cZGBfMnuyRbo1Y3H5rEQHiDwhqiE,5722
+interrater/data_objects/ratings.py,sha256=H_PguBmhXwauDghLlA4L_cmhoNjLVDu8BxFjubY2wPI,7529
+interrater/semantics/__init__.py,sha256=yOahD7VTbSa4Wp-EiNPpO2VYVLt-Ie8vABkBEjFM2QM,395
+interrater/semantics/categories.py,sha256=6asxTX376oAFou6REvPk6lLKyhNGgdBlGZhRupnX1as,11659
+interrater/semantics/ratings_semantics.py,sha256=WFM-79lISHk2lF3IHcJMrNf2o3EJJSby_HTBDSb7P6k,9943
+interrater-0.0.1.dev0.dist-info/METADATA,sha256=FguP1Bfy2hhqK0CHERpoXBooNCLOdTElG-QQ9HjqcJ8,3598
+interrater-0.0.1.dev0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+interrater-0.0.1.dev0.dist-info/licenses/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+interrater-0.0.1.dev0.dist-info/RECORD,,

interrater-0.0.1.dev0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any