reliability-lab 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/CHANGELOG.md +8 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/PKG-INFO +2 -2
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/README.md +1 -1
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/__init__.py +3 -1
- reliability_lab-0.1.4/src/reliability_lab/_version.py +1 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/alpha.py +47 -1
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/study.py +10 -2
- reliability_lab-0.1.4/tests/fixtures/krippendorff2011.py +153 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/reference.py +8 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_alpha.py +1 -1
- reliability_lab-0.1.4/tests/test_circular_bipolar.py +121 -0
- reliability_lab-0.1.4/tests/test_krippendorff2011.py +158 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_properties.py +1 -1
- reliability_lab-0.1.3/src/reliability_lab/_version.py +0 -1
- reliability_lab-0.1.3/tests/fixtures/krippendorff2011.py +0 -52
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.gitattributes +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.github/workflows/ci.yml +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.github/workflows/release.yml +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.gitignore +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.python-version +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/LICENSE +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/design/framing-v2.md +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/bootstrap-coverage.md +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-dark.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-light.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-dark.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-light.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-dark.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-light.png +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/releasing.md +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/llm_judge.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/quickstart.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/splits.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/pyproject.toml +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/__init__.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/coders.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/coverage.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/grid.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/report.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coders/coverage.csv +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coders/run_info.json +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coverage.csv +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/run_info.json +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/run_coverage.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/templates/bootstrap-coverage.md +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/data.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/design.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/exceptions.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/inference.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/llm.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/py.typed +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/report.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/split.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/fixtures/__init__.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/fixtures/llm_logs.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/helpers.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/naive_bootstrap.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_bca.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_coder_jackknife.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_coder_resampling.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_compare.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_data.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_design.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_inference_units.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_kalpha.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_llm_end_to_end.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_parity.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_report.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_report_io.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_retries.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_simulations.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_smoke.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_split.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_status_policies.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_trace.py +0 -0
- {reliability_lab-0.1.3 → reliability_lab-0.1.4}/uv.lock +0 -0
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.1.4
|
|
6
|
+
|
|
7
|
+
All six of Krippendorff's difference functions.
|
|
8
|
+
|
|
9
|
+
- Two more built-in distances from Krippendorff (2011):
|
|
10
|
+
- `measurement="bipolar"` is for scales with two poles, such as −3…+3. Its poles are the smallest and largest declared values.
|
|
11
|
+
- `measurement=circular(U)` is for values on a circle of circumference U: hours (24), degrees (360), months (12). U must be declared, so the string `"circular"` is refused.
|
|
12
|
+
|
|
5
13
|
## 0.1.3
|
|
6
14
|
|
|
7
15
|
Faster bootstrap.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: reliability-lab
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
|
|
5
5
|
Project-URL: Repository, https://github.com/Sathapana/reliability-lab
|
|
6
6
|
Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
|
|
@@ -31,7 +31,7 @@ Description-Content-Type: text/markdown
|
|
|
31
31
|
|
|
32
32
|
# reliability-lab
|
|
33
33
|
|
|
34
|
-
**Status: alpha (0.1.
|
|
34
|
+
**Status: alpha (0.1.4).** The API may still change between releases.
|
|
35
35
|
|
|
36
36
|
reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
|
|
37
37
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# reliability-lab
|
|
2
2
|
|
|
3
|
-
**Status: alpha (0.1.
|
|
3
|
+
**Status: alpha (0.1.4).** The API may still change between releases.
|
|
4
4
|
|
|
5
5
|
reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
|
|
6
6
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""reliability-lab: study design for the reliability of LLM and human judgments."""
|
|
2
2
|
|
|
3
3
|
from reliability_lab._version import __version__
|
|
4
|
-
from reliability_lab.alpha import INTERVAL, NOMINAL, ORDINAL, RATIO, Distance
|
|
4
|
+
from reliability_lab.alpha import BIPOLAR, INTERVAL, NOMINAL, ORDINAL, RATIO, Distance, circular
|
|
5
5
|
from reliability_lab.data import RatingFrame, Status
|
|
6
6
|
from reliability_lab.design import CoderSpec, Design
|
|
7
7
|
from reliability_lab.exceptions import (
|
|
@@ -33,6 +33,7 @@ from reliability_lab.split import SplitManifest
|
|
|
33
33
|
from reliability_lab.study import Fold, ReliabilityStudy
|
|
34
34
|
|
|
35
35
|
__all__ = [
|
|
36
|
+
"BIPOLAR",
|
|
36
37
|
"INTERVAL",
|
|
37
38
|
"NOMINAL",
|
|
38
39
|
"ORDINAL",
|
|
@@ -67,4 +68,5 @@ __all__ = [
|
|
|
67
68
|
"TraceWarning",
|
|
68
69
|
"UndefinedAlphaError",
|
|
69
70
|
"__version__",
|
|
71
|
+
"circular",
|
|
70
72
|
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.4"
|
|
@@ -13,11 +13,16 @@ on the distance. Precomputing them once therefore serves every distance and ever
|
|
|
13
13
|
bootstrap replicate. Only distances that depend on the marginals (ordinal) must be
|
|
14
14
|
rebuilt from each replicate's ``n_c``.
|
|
15
15
|
|
|
16
|
+
The built-in distances (nominal, ordinal, interval, ratio, bipolar and circular)
|
|
17
|
+
are the difference functions of that paper.
|
|
18
|
+
|
|
16
19
|
This module works on integer-coded numpy arrays only; ``study.py`` does the coding.
|
|
17
20
|
"""
|
|
18
21
|
|
|
19
22
|
from __future__ import annotations
|
|
20
23
|
|
|
24
|
+
import math
|
|
25
|
+
import numbers
|
|
21
26
|
from collections.abc import Callable, Sequence
|
|
22
27
|
from dataclasses import dataclass
|
|
23
28
|
from typing import Any
|
|
@@ -124,12 +129,53 @@ def _ratio(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
|
|
|
124
129
|
return np.asarray(np.where(total == 0, 0.0, (diff / safe) ** 2), dtype=float)
|
|
125
130
|
|
|
126
131
|
|
|
132
|
+
def _bipolar(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
|
|
133
|
+
# delta² = (c - k)² / ((c + k - 2 c_min)(2 c_max - c - k)), with c_min and c_max
|
|
134
|
+
# the ends of the declared scale. The denominator is zero only where c = k at an
|
|
135
|
+
# end, where the distance is zero.
|
|
136
|
+
v = values.astype(float)
|
|
137
|
+
c, k = v[:, None], v[None, :]
|
|
138
|
+
denominator = (c + k - 2 * v.min()) * (2 * v.max() - c - k)
|
|
139
|
+
safe = np.where(denominator == 0, 1.0, denominator)
|
|
140
|
+
return np.asarray(np.where(c == k, 0.0, (c - k) ** 2 / safe), dtype=float)
|
|
141
|
+
|
|
142
|
+
|
|
127
143
|
NOMINAL = Distance("nominal", _nominal)
|
|
128
144
|
ORDINAL = Distance("ordinal", _ordinal, depends_on_marginals=True)
|
|
129
145
|
INTERVAL = Distance("interval", _interval, numeric=True)
|
|
130
146
|
RATIO = Distance("ratio", _ratio, numeric=True)
|
|
147
|
+
BIPOLAR = Distance("bipolar", _bipolar, numeric=True)
|
|
148
|
+
|
|
149
|
+
BUILTIN_DISTANCES: dict[str, Distance] = {
|
|
150
|
+
d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO, BIPOLAR)
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def circular(circumference: float) -> Distance:
|
|
155
|
+
"""The circular distance sin²(π (c - k) / U) for values on a circle.
|
|
156
|
+
|
|
157
|
+
``circumference`` is U, the length of one full turn in the values' units: 24
|
|
158
|
+
for hours of the day, 360 for degrees, 12 for months, 7 for days of the week.
|
|
159
|
+
The values alone cannot fix it (the last hour of the day may never be rated),
|
|
160
|
+
so it is always declared.
|
|
161
|
+
"""
|
|
162
|
+
if (
|
|
163
|
+
isinstance(circumference, bool)
|
|
164
|
+
or not isinstance(circumference, numbers.Real)
|
|
165
|
+
or not math.isfinite(circumference)
|
|
166
|
+
or circumference <= 0
|
|
167
|
+
):
|
|
168
|
+
raise DesignError(
|
|
169
|
+
f"the circumference must be a finite number above 0, got {circumference!r}"
|
|
170
|
+
)
|
|
171
|
+
u = float(circumference)
|
|
172
|
+
label = str(int(u)) if u.is_integer() else repr(u)
|
|
173
|
+
|
|
174
|
+
def fn(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
|
|
175
|
+
v = values.astype(float)
|
|
176
|
+
return np.asarray(np.sin(np.pi * (v[:, None] - v[None, :]) / u) ** 2, dtype=float)
|
|
131
177
|
|
|
132
|
-
|
|
178
|
+
return Distance(f"circular({label})", fn, numeric=True)
|
|
133
179
|
|
|
134
180
|
|
|
135
181
|
def check_domain_size(n_values: int) -> str | None:
|
|
@@ -116,8 +116,11 @@ class ReliabilityStudy:
|
|
|
116
116
|
ratings:
|
|
117
117
|
Validated long-form ratings.
|
|
118
118
|
measurement:
|
|
119
|
-
``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``,
|
|
120
|
-
:class:`~reliability_lab.alpha.Distance`.
|
|
119
|
+
``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``, ``"bipolar"``,
|
|
120
|
+
``circular(U)``, or a :class:`~reliability_lab.alpha.Distance`. Bipolar
|
|
121
|
+
distances take the scale's two poles from the smallest and largest declared
|
|
122
|
+
values. Circular distances need the circumference U (24 for hours, 360 for
|
|
123
|
+
degrees), so the string ``"circular"`` is refused.
|
|
121
124
|
values:
|
|
122
125
|
The legal values. For ordinal data their order is the scale order. The
|
|
123
126
|
domain is always declared: pass ``ratings.observed_values()`` to declare the
|
|
@@ -935,6 +938,11 @@ def _resolve_distance(measurement: str | Distance) -> Distance:
|
|
|
935
938
|
return measurement
|
|
936
939
|
if isinstance(measurement, str) and measurement in BUILTIN_DISTANCES:
|
|
937
940
|
return BUILTIN_DISTANCES[measurement]
|
|
941
|
+
if measurement == "circular":
|
|
942
|
+
raise DesignError(
|
|
943
|
+
"circular data need the circle's circumference: pass measurement=circular(U), "
|
|
944
|
+
"for example circular(24) for hours or circular(360) for degrees"
|
|
945
|
+
)
|
|
938
946
|
raise DesignError(
|
|
939
947
|
f"measurement must be one of {list(BUILTIN_DISTANCES)} or a Distance; got {measurement!r}"
|
|
940
948
|
)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Worked examples from Krippendorff (2011), *Computing Krippendorff's alpha-reliability*.
|
|
2
|
+
|
|
3
|
+
Matrices are coders x units; ``None`` marks a missing rating. Data, coincidence
|
|
4
|
+
matrices, difference tables and alphas are transcribed from the paper (checked
|
|
5
|
+
against its text on 26 and 29 Oct 2026). The paper prints alpha to three decimals;
|
|
6
|
+
``exact_alpha``, ``d_o`` and ``d_e`` are the same quantities as fractions, computed
|
|
7
|
+
from the paper's formulas and matching its printed values.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from fractions import Fraction
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
THIRD = Fraction(1, 3)
|
|
14
|
+
|
|
15
|
+
#: Section A: binary data, two observers, no missing data.
|
|
16
|
+
BINARY_TWO_CODERS: dict[str, Any] = {
|
|
17
|
+
"matrix": {
|
|
18
|
+
"Meg": [0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
|
|
19
|
+
"Owen": [1, 1, 1, 0, 0, 1, 0, 0, 0, 0],
|
|
20
|
+
},
|
|
21
|
+
"values": [0, 1],
|
|
22
|
+
"n_pairable": 20,
|
|
23
|
+
"coincidence": [[10, 4], [4, 2]],
|
|
24
|
+
"marginals": [14, 6],
|
|
25
|
+
"alpha": {"nominal": 0.095},
|
|
26
|
+
# 1 - (n - 1) o_01 / (n_0 n_1) = 1 - 19 * 4 / (14 * 6)
|
|
27
|
+
"exact_alpha": {"nominal": Fraction(2, 21)},
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
#: Section B: nominal data, two observers, no missing data.
|
|
31
|
+
NOMINAL_TWO_CODERS: dict[str, Any] = {
|
|
32
|
+
"matrix": {
|
|
33
|
+
"Ben": list("aabbdcccedda"),
|
|
34
|
+
"Gerry": list("babbbccceddd"),
|
|
35
|
+
},
|
|
36
|
+
"values": ["a", "b", "c", "d", "e"],
|
|
37
|
+
"n_pairable": 24,
|
|
38
|
+
"coincidence": [
|
|
39
|
+
[2, 1, 0, 1, 0],
|
|
40
|
+
[1, 4, 0, 1, 0],
|
|
41
|
+
[0, 0, 6, 0, 0],
|
|
42
|
+
[1, 1, 0, 4, 0],
|
|
43
|
+
[0, 0, 0, 0, 2],
|
|
44
|
+
],
|
|
45
|
+
"marginals": [4, 6, 6, 6, 2],
|
|
46
|
+
"alpha": {"nominal": 0.692},
|
|
47
|
+
# (23 * 18 - 104) / (24 * 23 - 104)
|
|
48
|
+
"exact_alpha": {"nominal": Fraction(155, 224)},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
#: Sections C and D: four observers, twelve units, seven missing values. Unit 12
|
|
52
|
+
#: has a single value (from observer B), so it cannot be paired.
|
|
53
|
+
FOUR_CODERS_MISSING: dict[str, Any] = {
|
|
54
|
+
"matrix": {
|
|
55
|
+
"A": [1, 2, 3, 3, 2, 1, 4, 1, 2, None, None, None],
|
|
56
|
+
"B": [1, 2, 3, 3, 2, 2, 4, 1, 2, 5, None, 3],
|
|
57
|
+
"C": [None, 3, 3, 3, 2, 3, 4, 2, 2, 5, 1, None],
|
|
58
|
+
"D": [1, 2, 3, 3, 2, 4, 4, 1, 2, 5, 1, None],
|
|
59
|
+
},
|
|
60
|
+
"values": [1, 2, 3, 4, 5],
|
|
61
|
+
"n_pairable": 40,
|
|
62
|
+
"coincidence": [
|
|
63
|
+
[7, 4 * THIRD, THIRD, THIRD, 0],
|
|
64
|
+
[4 * THIRD, 10, 4 * THIRD, THIRD, 0],
|
|
65
|
+
[THIRD, 4 * THIRD, 8, THIRD, 0],
|
|
66
|
+
[THIRD, THIRD, THIRD, 4, 0],
|
|
67
|
+
[0, 0, 0, 0, 3],
|
|
68
|
+
],
|
|
69
|
+
"marginals": [9, 13, 10, 5, 3],
|
|
70
|
+
"alpha": {"nominal": 0.743, "ordinal": 0.815, "interval": 0.849, "ratio": 0.797},
|
|
71
|
+
"exact_alpha": {
|
|
72
|
+
"nominal": Fraction(113, 152),
|
|
73
|
+
"ordinal": Fraction(108577, 133160),
|
|
74
|
+
"interval": Fraction(951, 1120),
|
|
75
|
+
"ratio": Fraction(18222619, 22852465),
|
|
76
|
+
},
|
|
77
|
+
"d_o": {
|
|
78
|
+
"nominal": Fraction(1, 5),
|
|
79
|
+
"ordinal": Fraction(1891, 40),
|
|
80
|
+
"interval": Fraction(13, 30),
|
|
81
|
+
"ratio": Fraction(59357, 2646000),
|
|
82
|
+
},
|
|
83
|
+
"d_e": {
|
|
84
|
+
"nominal": Fraction(152, 195),
|
|
85
|
+
"ordinal": Fraction(3329, 13),
|
|
86
|
+
"interval": Fraction(112, 39),
|
|
87
|
+
"ratio": Fraction(4570493, 41277600),
|
|
88
|
+
},
|
|
89
|
+
# Section E: values (rows 1-5) by units (columns 1-12), n_uc, and the number
|
|
90
|
+
# of values in each unit, n_u.
|
|
91
|
+
"values_by_units": [
|
|
92
|
+
[3, 0, 0, 0, 0, 1, 0, 3, 0, 0, 2, 0],
|
|
93
|
+
[0, 3, 0, 0, 4, 1, 0, 1, 4, 0, 0, 0],
|
|
94
|
+
[0, 1, 4, 4, 0, 1, 0, 0, 0, 0, 0, 1],
|
|
95
|
+
[0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0],
|
|
96
|
+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0],
|
|
97
|
+
],
|
|
98
|
+
"values_per_unit": [3, 4, 4, 4, 4, 4, 4, 4, 4, 3, 2, 1],
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
#: Section D, page 6: ordinal differences delta (before squaring) for the
|
|
102
|
+
#: frequencies of example C with an unused 5th rank added. The paper prints the
|
|
103
|
+
#: upper triangle as delta squared and the lower triangle as its value; its lower
|
|
104
|
+
#: cell (5th, 1st) reads 992, but 32.5 squared is 1056.25, as its upper triangle
|
|
105
|
+
#: and formula give.
|
|
106
|
+
ORDINAL_DIFFERENCES: dict[str, Any] = {
|
|
107
|
+
"ranks": [1, 2, 3, 4, 5, 6],
|
|
108
|
+
"frequencies": [9, 13, 10, 5, 0, 3],
|
|
109
|
+
"delta": {
|
|
110
|
+
(1, 2): 11, (1, 3): 22.5, (1, 4): 30, (1, 5): 32.5, (1, 6): 34,
|
|
111
|
+
(2, 3): 11.5, (2, 4): 19, (2, 5): 21.5, (2, 6): 23,
|
|
112
|
+
(3, 4): 7.5, (3, 5): 10, (3, 6): 11.5,
|
|
113
|
+
(4, 5): 2.5, (4, 6): 4,
|
|
114
|
+
(5, 6): 1.5,
|
|
115
|
+
},
|
|
116
|
+
} # fmt: skip
|
|
117
|
+
|
|
118
|
+
#: Section D, page 7: ratio differences ((c - k) / (c + k))^2, printed to two decimals.
|
|
119
|
+
RATIO_DIFFERENCES: dict[str, Any] = {
|
|
120
|
+
"values": [0, 1, 2, 3, 4, 5],
|
|
121
|
+
"printed": {
|
|
122
|
+
(1, 0): 1, (2, 0): 1, (3, 0): 1, (4, 0): 1, (5, 0): 1,
|
|
123
|
+
(2, 1): 0.11, (3, 1): 0.25, (4, 1): 0.36, (5, 1): 0.44,
|
|
124
|
+
(3, 2): 0.04, (4, 2): 0.11, (5, 2): 0.18,
|
|
125
|
+
(4, 3): 0.02, (5, 3): 0.06,
|
|
126
|
+
(5, 4): 0.01,
|
|
127
|
+
},
|
|
128
|
+
} # fmt: skip
|
|
129
|
+
|
|
130
|
+
#: Section D, page 7: circular differences sin^2(180 (c - k) / U) with U = 6.
|
|
131
|
+
CIRCULAR_DIFFERENCES: dict[str, Any] = {
|
|
132
|
+
"values": [0, 1, 2, 3, 4, 5],
|
|
133
|
+
"circumference": 6,
|
|
134
|
+
"first_row": [0, 0.25, 0.75, 1, 0.75, 0.25],
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
#: Section D, page 7: bipolar differences (c - k)^2 / ((c + k - 2 c_min)(2 c_max - c - k)),
|
|
138
|
+
#: printed to three decimals.
|
|
139
|
+
BIPOLAR_DIFFERENCES: dict[str, Any] = {
|
|
140
|
+
"values": [-2, -1, 0, 1, 2],
|
|
141
|
+
"printed": {
|
|
142
|
+
(-1, -2): 0.143, (0, -2): 0.333, (1, -2): 0.600, (2, -2): 1.0,
|
|
143
|
+
(0, -1): 0.067, (1, -1): 0.250, (2, -1): 0.600,
|
|
144
|
+
(1, 0): 0.067, (2, 0): 0.333,
|
|
145
|
+
(2, 1): 0.143,
|
|
146
|
+
},
|
|
147
|
+
} # fmt: skip
|
|
148
|
+
|
|
149
|
+
EXAMPLES = {
|
|
150
|
+
"binary_two_coders": BINARY_TWO_CODERS,
|
|
151
|
+
"nominal_two_coders": NOMINAL_TWO_CODERS,
|
|
152
|
+
"four_coders_missing": FOUR_CODERS_MISSING,
|
|
153
|
+
}
|
|
@@ -64,6 +64,14 @@ def components_from_coincidences(
|
|
|
64
64
|
i, j = sorted((order.index(a), order.index(b)))
|
|
65
65
|
between = sum(n_c[order[g]] for g in range(i, j + 1))
|
|
66
66
|
return float((between - (n_c[a] + n_c[b]) / 2) ** 2)
|
|
67
|
+
if level == "bipolar":
|
|
68
|
+
if a == b:
|
|
69
|
+
return 0.0
|
|
70
|
+
low, high = min(order), max(order)
|
|
71
|
+
return float((a - b) ** 2 / ((a + b - 2 * low) * (2 * high - a - b)))
|
|
72
|
+
if level.startswith("circular(") and level.endswith(")"):
|
|
73
|
+
circumference = float(level[len("circular(") : -1])
|
|
74
|
+
return math.sin(math.pi * (a - b) / circumference) ** 2
|
|
67
75
|
raise ValueError(level)
|
|
68
76
|
|
|
69
77
|
if n == 0:
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""The built-in bipolar and circular distances (Krippendorff 2011, section D)."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pytest
|
|
9
|
+
from helpers import human_study, long_from_matrix, random_matrix
|
|
10
|
+
from reference import reference_alpha
|
|
11
|
+
|
|
12
|
+
from reliability_lab import BIPOLAR, DesignError, RatingFrame, ReliabilityStudy, circular
|
|
13
|
+
|
|
14
|
+
pytestmark = pytest.mark.filterwarnings("ignore::reliability_lab.CategorySkewWarning")
|
|
15
|
+
|
|
16
|
+
HOURS = list(range(24))
|
|
17
|
+
#: (oracle level, measurement, values)
|
|
18
|
+
CASES = [
|
|
19
|
+
("bipolar", "bipolar", [-2, -1, 0, 1, 2]),
|
|
20
|
+
("circular(12)", circular(12), list(range(1, 13))),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.mark.parametrize(("level", "measurement", "values"), CASES, ids=["bipolar", "circular"])
|
|
25
|
+
def test_engine_matches_reference_on_random_data(
|
|
26
|
+
level: str, measurement: Any, values: list[int]
|
|
27
|
+
) -> None:
|
|
28
|
+
rng = np.random.default_rng(2029)
|
|
29
|
+
checked = 0
|
|
30
|
+
for _ in range(25):
|
|
31
|
+
matrix = random_matrix(
|
|
32
|
+
rng, int(rng.integers(2, 6)), int(rng.integers(3, 30)), values, p_missing=0.25
|
|
33
|
+
)
|
|
34
|
+
expected = reference_alpha(matrix, level, values)
|
|
35
|
+
if math.isnan(expected):
|
|
36
|
+
continue
|
|
37
|
+
assert human_study(matrix, measurement, values).alpha().alpha == pytest.approx(
|
|
38
|
+
expected, abs=1e-12
|
|
39
|
+
)
|
|
40
|
+
checked += 1
|
|
41
|
+
assert checked >= 20
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_circular_needs_its_circumference() -> None:
|
|
45
|
+
with pytest.raises(DesignError, match=r"circular\(U\)"):
|
|
46
|
+
human_study({"a": [1, 2], "b": [1, 2]}, "circular", [1, 2])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.parametrize("circumference", [0, -1, math.nan, math.inf, True, "24", None])
|
|
50
|
+
def test_circumference_must_be_a_positive_number(circumference: Any) -> None:
|
|
51
|
+
with pytest.raises(DesignError, match="circumference"):
|
|
52
|
+
circular(circumference)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_the_circumference_is_recorded() -> None:
|
|
56
|
+
matrix = {"a": [0, 6, 12, 23], "b": [1, 6, 13, 0]}
|
|
57
|
+
result = human_study(matrix, circular(24), HOURS).alpha()
|
|
58
|
+
assert result.measurement == "circular(24)"
|
|
59
|
+
payload = json.loads(result.to_json())
|
|
60
|
+
assert payload["measurement"] == "circular(24)"
|
|
61
|
+
assert payload["provenance"]["declarations"]["measurement"] == "circular(24)"
|
|
62
|
+
assert circular(6.5).name == "circular(6.5)"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_hours_wrap_around_midnight() -> None:
|
|
66
|
+
d = circular(24).matrix(HOURS, np.ones(24))
|
|
67
|
+
assert d[23, 0] == pytest.approx(d[0, 1])
|
|
68
|
+
assert d[0, 12] == pytest.approx(1.0) # opposite hours are the farthest apart
|
|
69
|
+
assert d.max() == pytest.approx(1.0)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_rotating_every_value_leaves_circular_alpha_unchanged() -> None:
|
|
73
|
+
rng = np.random.default_rng(7)
|
|
74
|
+
matrix = random_matrix(rng, 4, 30, HOURS, p_missing=0.2)
|
|
75
|
+
rotated = {
|
|
76
|
+
coder: [None if v is None else (v + 5) % 24 for v in column]
|
|
77
|
+
for coder, column in matrix.items()
|
|
78
|
+
}
|
|
79
|
+
a = human_study(matrix, circular(24), HOURS).alpha().alpha
|
|
80
|
+
b = human_study(rotated, circular(24), HOURS).alpha().alpha
|
|
81
|
+
assert a == pytest.approx(b, abs=1e-12)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_bipolar_takes_its_poles_from_the_declared_values() -> None:
|
|
85
|
+
narrow = BIPOLAR.matrix([-2, -1, 0, 1, 2], np.ones(5))
|
|
86
|
+
wide = BIPOLAR.matrix([-3, -2, -1, 0, 1, 2, 3], np.ones(7))
|
|
87
|
+
# -2 and 2 are the poles of the narrow scale, the farthest apart possible ...
|
|
88
|
+
assert narrow[0, 4] == pytest.approx(1.0)
|
|
89
|
+
# ... but on a -3..3 scale they are 16 / (6 * 6) apart.
|
|
90
|
+
assert wide[1, 5] == pytest.approx(16 / 36)
|
|
91
|
+
with pytest.raises(DesignError, match="numeric"):
|
|
92
|
+
human_study({"a": ["x", "y"], "b": ["x", "y"]}, "bipolar", ["x", "y"])
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@pytest.mark.parametrize(("level", "measurement", "values"), CASES, ids=["bipolar", "circular"])
|
|
96
|
+
def test_bootstraps_and_sensitivity_run(level: str, measurement: Any, values: list[int]) -> None:
|
|
97
|
+
rng = np.random.default_rng(3)
|
|
98
|
+
# Six coders: with four, a two-way resample now and then draws one coder four
|
|
99
|
+
# times, leaving no pairs (counted and warned about, but noise in this test).
|
|
100
|
+
frame = long_from_matrix(random_matrix(rng, 6, 40, values, p_missing=0.1))
|
|
101
|
+
frame["status"] = "observed"
|
|
102
|
+
frame.loc[0, ["value", "status"]] = [None, "parse_error"]
|
|
103
|
+
ratings = RatingFrame.from_long(
|
|
104
|
+
frame, unit="unit", rater="coder", value="value", status="status"
|
|
105
|
+
)
|
|
106
|
+
study = ReliabilityStudy(
|
|
107
|
+
ratings, measurement=measurement, values=values, coders="rater_id", design="human_panel"
|
|
108
|
+
)
|
|
109
|
+
result = study.alpha()
|
|
110
|
+
assert set(result.estimates) == {
|
|
111
|
+
"as_missing",
|
|
112
|
+
"sensitivity_pessimistic",
|
|
113
|
+
"sensitivity_optimistic",
|
|
114
|
+
}
|
|
115
|
+
pessimistic, optimistic = (
|
|
116
|
+
result.estimates[f"sensitivity_{v}"].alpha for v in ("pessimistic", "optimistic")
|
|
117
|
+
)
|
|
118
|
+
assert pessimistic <= optimistic
|
|
119
|
+
for scheme in ("units", "two_way"):
|
|
120
|
+
boot = result.bootstrap(scheme=scheme, n_resamples=200, random_state=1) # type: ignore[arg-type]
|
|
121
|
+
assert boot["as_missing"].defined
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""The intermediate quantities Krippendorff (2011) prints, not just its alphas:
|
|
2
|
+
coincidence matrices, per-unit contributions, difference tables, D_o and D_e, and
|
|
3
|
+
the values-by-units form of section E."""
|
|
4
|
+
|
|
5
|
+
from itertools import combinations
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pytest
|
|
10
|
+
from fixtures.krippendorff2011 import (
|
|
11
|
+
BIPOLAR_DIFFERENCES,
|
|
12
|
+
CIRCULAR_DIFFERENCES,
|
|
13
|
+
EXAMPLES,
|
|
14
|
+
FOUR_CODERS_MISSING,
|
|
15
|
+
ORDINAL_DIFFERENCES,
|
|
16
|
+
RATIO_DIFFERENCES,
|
|
17
|
+
)
|
|
18
|
+
from helpers import human_study
|
|
19
|
+
|
|
20
|
+
from reliability_lab import BIPOLAR, Distance, circular
|
|
21
|
+
from reliability_lab.alpha import BUILTIN_DISTANCES, INTERVAL, ORDINAL, RATIO, per_unit_coincidences
|
|
22
|
+
|
|
23
|
+
F4 = FOUR_CODERS_MISSING
|
|
24
|
+
LEVELS = ["nominal", "ordinal", "interval", "ratio"]
|
|
25
|
+
EXACT = [
|
|
26
|
+
(name, level, exact)
|
|
27
|
+
for name, example in EXAMPLES.items()
|
|
28
|
+
for level, exact in example["exact_alpha"].items()
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _as_float(matrix: list[list[Any]]) -> np.ndarray:
|
|
33
|
+
return np.array([[float(x) for x in row] for row in matrix])
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@pytest.mark.parametrize("name", list(EXAMPLES))
|
|
37
|
+
def test_coincidence_matrix_matches_the_paper(name: str) -> None:
|
|
38
|
+
example = EXAMPLES[name]
|
|
39
|
+
est = human_study(example["matrix"], "nominal", example["values"]).alpha()["as_missing"]
|
|
40
|
+
np.testing.assert_allclose(
|
|
41
|
+
est.coincidence.to_numpy(), _as_float(example["coincidence"]), atol=1e-12
|
|
42
|
+
)
|
|
43
|
+
np.testing.assert_allclose(est.coincidence.to_numpy().sum(axis=1), example["marginals"])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@pytest.mark.parametrize(("name", "level", "exact"), EXACT)
|
|
47
|
+
def test_alpha_matches_the_exact_value(name: str, level: str, exact: Any) -> None:
|
|
48
|
+
example = EXAMPLES[name]
|
|
49
|
+
result = human_study(example["matrix"], level, example["values"]).alpha()
|
|
50
|
+
assert result.alpha == pytest.approx(float(exact), abs=1e-12)
|
|
51
|
+
assert round(float(exact), 3) == example["alpha"][level]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.mark.parametrize("level", LEVELS)
|
|
55
|
+
def test_observed_and_expected_disagreement(level: str) -> None:
|
|
56
|
+
est = human_study(F4["matrix"], level, F4["values"]).alpha()["as_missing"]
|
|
57
|
+
assert est.d_o == pytest.approx(float(F4["d_o"][level]), rel=1e-12)
|
|
58
|
+
assert est.d_e == pytest.approx(float(F4["d_e"][level]), rel=1e-12)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_unit_contributions_described_in_the_text() -> None:
|
|
62
|
+
units, codes = [], []
|
|
63
|
+
for u, column in enumerate(zip(*F4["matrix"].values(), strict=True)):
|
|
64
|
+
for value in column:
|
|
65
|
+
if value is not None:
|
|
66
|
+
units.append(u)
|
|
67
|
+
codes.append(F4["values"].index(value))
|
|
68
|
+
per_unit, pairable = per_unit_coincidences(np.array(units), np.array(codes), 12, 5)
|
|
69
|
+
o = per_unit.toarray().reshape(12, 5, 5)
|
|
70
|
+
# Unit 1: three 1s, 6 pairs, adds 6 / (3 - 1) = 3 to o_11.
|
|
71
|
+
assert o[0, 0, 0] == pytest.approx(3) and o[0].sum() == pytest.approx(3)
|
|
72
|
+
# Unit 2: adds 2 to o_22, 1 to o_23 and 1 to o_32, accounting for its 4 values.
|
|
73
|
+
assert (o[1, 1, 1], o[1, 1, 2], o[1, 2, 1]) == pytest.approx((2, 1, 1))
|
|
74
|
+
assert o[1].sum() == pytest.approx(4)
|
|
75
|
+
# Unit 6: 12 mismatching pairs, each adding 1/3 to a different cell.
|
|
76
|
+
mismatches = o[5][~np.eye(5, dtype=bool)]
|
|
77
|
+
assert np.count_nonzero(mismatches) == 12
|
|
78
|
+
np.testing.assert_allclose(mismatches[mismatches > 0], 1 / 3)
|
|
79
|
+
# Unit 12: a lone value, no pairs.
|
|
80
|
+
assert pairable[11] == 1 and o[11].sum() == 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@pytest.mark.parametrize("level", LEVELS)
|
|
84
|
+
def test_values_by_units_form(level: str) -> None:
|
|
85
|
+
"""Section E: alpha and each unit's share of D_o from the values-by-units counts."""
|
|
86
|
+
est = human_study(F4["matrix"], level, F4["values"]).alpha()["as_missing"]
|
|
87
|
+
table = est.per_unit.set_index("unit_id").sort_index()
|
|
88
|
+
counts = np.array(F4["values_by_units"], dtype=float)
|
|
89
|
+
per_unit = np.array(F4["values_per_unit"], dtype=float)
|
|
90
|
+
np.testing.assert_array_equal(table["pairable_values"], per_unit)
|
|
91
|
+
pairable = per_unit >= 2
|
|
92
|
+
totals = counts[:, pairable].sum(axis=1)
|
|
93
|
+
np.testing.assert_array_equal(totals, F4["marginals"])
|
|
94
|
+
n = totals.sum()
|
|
95
|
+
delta2 = BUILTIN_DISTANCES[level].matrix(F4["values"], totals)
|
|
96
|
+
pairs = list(combinations(range(len(totals)), 2))
|
|
97
|
+
within = np.array(
|
|
98
|
+
[
|
|
99
|
+
sum(counts[c, u] * counts[k, u] * delta2[c, k] for c, k in pairs) / (per_unit[u] - 1)
|
|
100
|
+
if pairable[u]
|
|
101
|
+
else 0.0
|
|
102
|
+
for u in range(counts.shape[1])
|
|
103
|
+
]
|
|
104
|
+
)
|
|
105
|
+
# A unit's share of D_o counts both orders of each pair.
|
|
106
|
+
np.testing.assert_allclose(table["d_o_contribution"], 2 * within / n, atol=1e-12)
|
|
107
|
+
between = sum(totals[c] * totals[k] * delta2[c, k] for c, k in pairs)
|
|
108
|
+
alpha = 1 - (n - 1) * within.sum() / between
|
|
109
|
+
assert alpha == pytest.approx(float(F4["exact_alpha"][level]), abs=1e-12)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_ordinal_differences_table() -> None:
|
|
113
|
+
table = ORDINAL_DIFFERENCES
|
|
114
|
+
delta2 = ORDINAL.matrix(table["ranks"], table["frequencies"])
|
|
115
|
+
for (c, k), delta in table["delta"].items():
|
|
116
|
+
assert delta2[c - 1, k - 1] == pytest.approx(delta**2)
|
|
117
|
+
assert delta2[k - 1, c - 1] == pytest.approx(delta**2)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_an_unused_rank_does_not_change_ordinal_alpha() -> None:
|
|
121
|
+
shifted = {coder: [6 if v == 5 else v for v in col] for coder, col in F4["matrix"].items()}
|
|
122
|
+
alpha = human_study(shifted, "ordinal", [1, 2, 3, 4, 5, 6]).alpha().alpha
|
|
123
|
+
assert alpha == pytest.approx(float(F4["exact_alpha"]["ordinal"]), abs=1e-12)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_standardised_ordinal_differences_give_the_same_alpha() -> None:
|
|
127
|
+
def standardised(values: Any, marginals: Any) -> Any:
|
|
128
|
+
d = ORDINAL.fn(values, marginals)
|
|
129
|
+
return d / d.max()
|
|
130
|
+
|
|
131
|
+
distance = Distance("ordinal_standardised", standardised, depends_on_marginals=True)
|
|
132
|
+
alpha = human_study(F4["matrix"], distance, F4["values"]).alpha().alpha
|
|
133
|
+
assert alpha == pytest.approx(float(F4["exact_alpha"]["ordinal"]), abs=1e-12)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_interval_and_ratio_tables() -> None:
|
|
137
|
+
values = np.arange(-1, 5)
|
|
138
|
+
expected = (values[:, None] - values[None, :]) ** 2
|
|
139
|
+
np.testing.assert_array_equal(INTERVAL.matrix(list(values), np.ones(6)), expected)
|
|
140
|
+
ratio = RATIO.matrix(RATIO_DIFFERENCES["values"], np.ones(6))
|
|
141
|
+
for (c, k), printed in RATIO_DIFFERENCES["printed"].items():
|
|
142
|
+
assert round(ratio[c, k], 2) == printed
|
|
143
|
+
assert ratio[k, c] == ratio[c, k]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_circular_and_bipolar_tables() -> None:
|
|
147
|
+
d = circular(CIRCULAR_DIFFERENCES["circumference"]).matrix(
|
|
148
|
+
CIRCULAR_DIFFERENCES["values"], np.ones(6)
|
|
149
|
+
)
|
|
150
|
+
np.testing.assert_allclose(d[0], CIRCULAR_DIFFERENCES["first_row"], atol=1e-12)
|
|
151
|
+
# Every row is the first row rotated: only the distance around the circle counts.
|
|
152
|
+
for c in range(6):
|
|
153
|
+
np.testing.assert_allclose(d[c], np.roll(d[0], c), atol=1e-12)
|
|
154
|
+
|
|
155
|
+
values = BIPOLAR_DIFFERENCES["values"]
|
|
156
|
+
d = BIPOLAR.matrix(values, np.ones(5))
|
|
157
|
+
for (c, k), printed in BIPOLAR_DIFFERENCES["printed"].items():
|
|
158
|
+
assert round(d[values.index(c), values.index(k)], 3) == printed
|
|
@@ -11,7 +11,7 @@ from reference import reference_alpha
|
|
|
11
11
|
|
|
12
12
|
from reliability_lab import RatingFrame, ReliabilityStudy, UndefinedAlphaError
|
|
13
13
|
|
|
14
|
-
LEVELS = ["nominal", "ordinal", "interval", "ratio"]
|
|
14
|
+
LEVELS = ["nominal", "ordinal", "interval", "ratio", "bipolar"]
|
|
15
15
|
|
|
16
16
|
# Random data often has one dominant category; the warning is expected here.
|
|
17
17
|
pytestmark = pytest.mark.filterwarnings("ignore::reliability_lab.CategorySkewWarning")
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.3"
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
"""Worked examples from Krippendorff (2011), *Computing Krippendorff's alpha-reliability*.
|
|
2
|
-
|
|
3
|
-
Matrices are coders x units; ``None`` marks a missing rating. Data and expected
|
|
4
|
-
values are transcribed from the paper (checked against its text on 26 Oct 2026).
|
|
5
|
-
The paper prints alpha to three decimals. The ratio value for example C did not
|
|
6
|
-
extract cleanly from the PDF, so it is also covered by the parity tests against
|
|
7
|
-
the ``krippendorff`` package.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from typing import Any
|
|
11
|
-
|
|
12
|
-
#: Section A: binary data, two observers, no missing data.
|
|
13
|
-
BINARY_TWO_CODERS: dict[str, Any] = {
|
|
14
|
-
"matrix": {
|
|
15
|
-
"Meg": [0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
|
|
16
|
-
"Owen": [1, 1, 1, 0, 0, 1, 0, 0, 0, 0],
|
|
17
|
-
},
|
|
18
|
-
"values": [0, 1],
|
|
19
|
-
"n_pairable": 20,
|
|
20
|
-
"alpha": {"nominal": 0.095},
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
#: Section B: nominal data, two observers, no missing data.
|
|
24
|
-
NOMINAL_TWO_CODERS: dict[str, Any] = {
|
|
25
|
-
"matrix": {
|
|
26
|
-
"Ben": list("aabbdcccedda"),
|
|
27
|
-
"Gerry": list("babbbccceddd"),
|
|
28
|
-
},
|
|
29
|
-
"values": ["a", "b", "c", "d", "e"],
|
|
30
|
-
"n_pairable": 24,
|
|
31
|
-
"alpha": {"nominal": 0.692},
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
#: Sections C and D: four observers, twelve units, seven missing values. Unit 12
|
|
35
|
-
#: has a single value (from observer B), so it cannot be paired.
|
|
36
|
-
FOUR_CODERS_MISSING: dict[str, Any] = {
|
|
37
|
-
"matrix": {
|
|
38
|
-
"A": [1, 2, 3, 3, 2, 1, 4, 1, 2, None, None, None],
|
|
39
|
-
"B": [1, 2, 3, 3, 2, 2, 4, 1, 2, 5, None, 3],
|
|
40
|
-
"C": [None, 3, 3, 3, 2, 3, 4, 2, 2, 5, 1, None],
|
|
41
|
-
"D": [1, 2, 3, 3, 2, 4, 4, 1, 2, 5, 1, None],
|
|
42
|
-
},
|
|
43
|
-
"values": [1, 2, 3, 4, 5],
|
|
44
|
-
"n_pairable": 40,
|
|
45
|
-
"alpha": {"nominal": 0.743, "ordinal": 0.815, "interval": 0.849, "ratio": 0.797},
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
EXAMPLES = {
|
|
49
|
-
"binary_two_coders": BINARY_TWO_CODERS,
|
|
50
|
-
"nominal_two_coders": NOMINAL_TWO_CODERS,
|
|
51
|
-
"four_coders_missing": FOUR_CODERS_MISSING,
|
|
52
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-dark.png
RENAMED
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-light.png
RENAMED
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-dark.png
RENAMED
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-light.png
RENAMED
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-dark.png
RENAMED
|
File without changes
|
{reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-light.png
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|