reliability-lab 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/CHANGELOG.md +8 -0
  2. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/PKG-INFO +2 -2
  3. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/README.md +1 -1
  4. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/__init__.py +3 -1
  5. reliability_lab-0.1.4/src/reliability_lab/_version.py +1 -0
  6. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/alpha.py +47 -1
  7. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/study.py +10 -2
  8. reliability_lab-0.1.4/tests/fixtures/krippendorff2011.py +153 -0
  9. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/reference.py +8 -0
  10. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_alpha.py +1 -1
  11. reliability_lab-0.1.4/tests/test_circular_bipolar.py +121 -0
  12. reliability_lab-0.1.4/tests/test_krippendorff2011.py +158 -0
  13. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_properties.py +1 -1
  14. reliability_lab-0.1.3/src/reliability_lab/_version.py +0 -1
  15. reliability_lab-0.1.3/tests/fixtures/krippendorff2011.py +0 -52
  16. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.gitattributes +0 -0
  17. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.github/workflows/ci.yml +0 -0
  18. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.github/workflows/release.yml +0 -0
  19. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.gitignore +0 -0
  20. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/.python-version +0 -0
  21. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/LICENSE +0 -0
  22. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/design/framing-v2.md +0 -0
  23. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/bootstrap-coverage.md +0 -0
  24. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-dark.png +0 -0
  25. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-light.png +0 -0
  26. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-dark.png +0 -0
  27. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-light.png +0 -0
  28. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-dark.png +0 -0
  29. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-light.png +0 -0
  30. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/docs/releasing.md +0 -0
  31. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/llm_judge.py +0 -0
  32. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/quickstart.py +0 -0
  33. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/examples/splits.py +0 -0
  34. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/pyproject.toml +0 -0
  35. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/__init__.py +0 -0
  36. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/coders.py +0 -0
  37. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/coverage.py +0 -0
  38. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/grid.py +0 -0
  39. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/report.py +0 -0
  40. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coders/coverage.csv +0 -0
  41. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coders/run_info.json +0 -0
  42. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/coverage.csv +0 -0
  43. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/results/run_info.json +0 -0
  44. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/run_coverage.py +0 -0
  45. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/simulations/templates/bootstrap-coverage.md +0 -0
  46. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/data.py +0 -0
  47. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/design.py +0 -0
  48. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/exceptions.py +0 -0
  49. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/inference.py +0 -0
  50. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/llm.py +0 -0
  51. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/py.typed +0 -0
  52. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/report.py +0 -0
  53. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/src/reliability_lab/split.py +0 -0
  54. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/fixtures/__init__.py +0 -0
  55. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/fixtures/llm_logs.py +0 -0
  56. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/helpers.py +0 -0
  57. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/naive_bootstrap.py +0 -0
  58. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_bca.py +0 -0
  59. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_coder_jackknife.py +0 -0
  60. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_coder_resampling.py +0 -0
  61. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_compare.py +0 -0
  62. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_data.py +0 -0
  63. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_design.py +0 -0
  64. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_inference_units.py +0 -0
  65. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_kalpha.py +0 -0
  66. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_llm_end_to_end.py +0 -0
  67. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_parity.py +0 -0
  68. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_report.py +0 -0
  69. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_report_io.py +0 -0
  70. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_retries.py +0 -0
  71. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_simulations.py +0 -0
  72. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_smoke.py +0 -0
  73. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_split.py +0 -0
  74. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_status_policies.py +0 -0
  75. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/tests/test_trace.py +0 -0
  76. {reliability_lab-0.1.3 → reliability_lab-0.1.4}/uv.lock +0 -0
@@ -2,6 +2,14 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ## 0.1.4
6
+
7
+ All six of Krippendorff's difference functions.
8
+
9
+ - Two more built-in distances from Krippendorff (2011):
10
+ - `measurement="bipolar"` is for scales with two poles, such as −3…+3. Its poles are the smallest and largest declared values.
11
+ - `measurement=circular(U)` is for values on a circle of circumference U: hours (24), degrees (360), months (12). U must be declared, so the string `"circular"` is refused.
12
+
5
13
  ## 0.1.3
6
14
 
7
15
  Faster bootstrap.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: reliability-lab
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
5
5
  Project-URL: Repository, https://github.com/Sathapana/reliability-lab
6
6
  Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
@@ -31,7 +31,7 @@ Description-Content-Type: text/markdown
31
31
 
32
32
  # reliability-lab
33
33
 
34
- **Status: alpha (0.1.3).** The API may still change between releases.
34
+ **Status: alpha (0.1.4).** The API may still change between releases.
35
35
 
36
36
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
37
37
 
@@ -1,6 +1,6 @@
1
1
  # reliability-lab
2
2
 
3
- **Status: alpha (0.1.3).** The API may still change between releases.
3
+ **Status: alpha (0.1.4).** The API may still change between releases.
4
4
 
5
5
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
6
6
 
@@ -1,7 +1,7 @@
1
1
  """reliability-lab: study design for the reliability of LLM and human judgments."""
2
2
 
3
3
  from reliability_lab._version import __version__
4
- from reliability_lab.alpha import INTERVAL, NOMINAL, ORDINAL, RATIO, Distance
4
+ from reliability_lab.alpha import BIPOLAR, INTERVAL, NOMINAL, ORDINAL, RATIO, Distance, circular
5
5
  from reliability_lab.data import RatingFrame, Status
6
6
  from reliability_lab.design import CoderSpec, Design
7
7
  from reliability_lab.exceptions import (
@@ -33,6 +33,7 @@ from reliability_lab.split import SplitManifest
33
33
  from reliability_lab.study import Fold, ReliabilityStudy
34
34
 
35
35
  __all__ = [
36
+ "BIPOLAR",
36
37
  "INTERVAL",
37
38
  "NOMINAL",
38
39
  "ORDINAL",
@@ -67,4 +68,5 @@ __all__ = [
67
68
  "TraceWarning",
68
69
  "UndefinedAlphaError",
69
70
  "__version__",
71
+ "circular",
70
72
  ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.4"
@@ -13,11 +13,16 @@ on the distance. Precomputing them once therefore serves every distance and ever
13
13
  bootstrap replicate. Only distances that depend on the marginals (ordinal) must be
14
14
  rebuilt from each replicate's ``n_c``.
15
15
 
16
+ The built-in distances (nominal, ordinal, interval, ratio, bipolar and circular)
17
+ are the difference functions of that paper.
18
+
16
19
  This module works on integer-coded numpy arrays only; ``study.py`` does the coding.
17
20
  """
18
21
 
19
22
  from __future__ import annotations
20
23
 
24
+ import math
25
+ import numbers
21
26
  from collections.abc import Callable, Sequence
22
27
  from dataclasses import dataclass
23
28
  from typing import Any
@@ -124,12 +129,53 @@ def _ratio(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
124
129
  return np.asarray(np.where(total == 0, 0.0, (diff / safe) ** 2), dtype=float)
125
130
 
126
131
 
132
+ def _bipolar(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
133
+ # delta² = (c - k)² / ((c + k - 2 c_min)(2 c_max - c - k)), with c_min and c_max
134
+ # the ends of the declared scale. The denominator is zero only where c = k at an
135
+ # end, where the distance is zero.
136
+ v = values.astype(float)
137
+ c, k = v[:, None], v[None, :]
138
+ denominator = (c + k - 2 * v.min()) * (2 * v.max() - c - k)
139
+ safe = np.where(denominator == 0, 1.0, denominator)
140
+ return np.asarray(np.where(c == k, 0.0, (c - k) ** 2 / safe), dtype=float)
141
+
142
+
127
143
  NOMINAL = Distance("nominal", _nominal)
128
144
  ORDINAL = Distance("ordinal", _ordinal, depends_on_marginals=True)
129
145
  INTERVAL = Distance("interval", _interval, numeric=True)
130
146
  RATIO = Distance("ratio", _ratio, numeric=True)
147
+ BIPOLAR = Distance("bipolar", _bipolar, numeric=True)
148
+
149
+ BUILTIN_DISTANCES: dict[str, Distance] = {
150
+ d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO, BIPOLAR)
151
+ }
152
+
153
+
154
+ def circular(circumference: float) -> Distance:
155
+ """The circular distance sin²(π (c - k) / U) for values on a circle.
156
+
157
+ ``circumference`` is U, the length of one full turn in the values' units: 24
158
+ for hours of the day, 360 for degrees, 12 for months, 7 for days of the week.
159
+ The values alone cannot fix it (the last hour of the day may never be rated),
160
+ so it is always declared.
161
+ """
162
+ if (
163
+ isinstance(circumference, bool)
164
+ or not isinstance(circumference, numbers.Real)
165
+ or not math.isfinite(circumference)
166
+ or circumference <= 0
167
+ ):
168
+ raise DesignError(
169
+ f"the circumference must be a finite number above 0, got {circumference!r}"
170
+ )
171
+ u = float(circumference)
172
+ label = str(int(u)) if u.is_integer() else repr(u)
173
+
174
+ def fn(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
175
+ v = values.astype(float)
176
+ return np.asarray(np.sin(np.pi * (v[:, None] - v[None, :]) / u) ** 2, dtype=float)
131
177
 
132
- BUILTIN_DISTANCES: dict[str, Distance] = {d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO)}
178
+ return Distance(f"circular({label})", fn, numeric=True)
133
179
 
134
180
 
135
181
  def check_domain_size(n_values: int) -> str | None:
@@ -116,8 +116,11 @@ class ReliabilityStudy:
116
116
  ratings:
117
117
  Validated long-form ratings.
118
118
  measurement:
119
- ``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``, or a
120
- :class:`~reliability_lab.alpha.Distance`.
119
+ ``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``, ``"bipolar"``,
120
+ ``circular(U)``, or a :class:`~reliability_lab.alpha.Distance`. Bipolar
121
+ distances take the scale's two poles from the smallest and largest declared
122
+ values. Circular distances need the circumference U (24 for hours, 360 for
123
+ degrees), so the string ``"circular"`` is refused.
121
124
  values:
122
125
  The legal values. For ordinal data their order is the scale order. The
123
126
  domain is always declared: pass ``ratings.observed_values()`` to declare the
@@ -935,6 +938,11 @@ def _resolve_distance(measurement: str | Distance) -> Distance:
935
938
  return measurement
936
939
  if isinstance(measurement, str) and measurement in BUILTIN_DISTANCES:
937
940
  return BUILTIN_DISTANCES[measurement]
941
+ if measurement == "circular":
942
+ raise DesignError(
943
+ "circular data need the circle's circumference: pass measurement=circular(U), "
944
+ "for example circular(24) for hours or circular(360) for degrees"
945
+ )
938
946
  raise DesignError(
939
947
  f"measurement must be one of {list(BUILTIN_DISTANCES)} or a Distance; got {measurement!r}"
940
948
  )
@@ -0,0 +1,153 @@
1
+ """Worked examples from Krippendorff (2011), *Computing Krippendorff's alpha-reliability*.
2
+
3
+ Matrices are coders x units; ``None`` marks a missing rating. Data, coincidence
4
+ matrices, difference tables and alphas are transcribed from the paper (checked
5
+ against its text on 26 and 29 Oct 2026). The paper prints alpha to three decimals;
6
+ ``exact_alpha``, ``d_o`` and ``d_e`` are the same quantities as fractions, computed
7
+ from the paper's formulas and matching its printed values.
8
+ """
9
+
10
+ from fractions import Fraction
11
+ from typing import Any
12
+
13
+ THIRD = Fraction(1, 3)
14
+
15
+ #: Section A: binary data, two observers, no missing data.
16
+ BINARY_TWO_CODERS: dict[str, Any] = {
17
+ "matrix": {
18
+ "Meg": [0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
19
+ "Owen": [1, 1, 1, 0, 0, 1, 0, 0, 0, 0],
20
+ },
21
+ "values": [0, 1],
22
+ "n_pairable": 20,
23
+ "coincidence": [[10, 4], [4, 2]],
24
+ "marginals": [14, 6],
25
+ "alpha": {"nominal": 0.095},
26
+ # 1 - (n - 1) o_01 / (n_0 n_1) = 1 - 19 * 4 / (14 * 6)
27
+ "exact_alpha": {"nominal": Fraction(2, 21)},
28
+ }
29
+
30
+ #: Section B: nominal data, two observers, no missing data.
31
+ NOMINAL_TWO_CODERS: dict[str, Any] = {
32
+ "matrix": {
33
+ "Ben": list("aabbdcccedda"),
34
+ "Gerry": list("babbbccceddd"),
35
+ },
36
+ "values": ["a", "b", "c", "d", "e"],
37
+ "n_pairable": 24,
38
+ "coincidence": [
39
+ [2, 1, 0, 1, 0],
40
+ [1, 4, 0, 1, 0],
41
+ [0, 0, 6, 0, 0],
42
+ [1, 1, 0, 4, 0],
43
+ [0, 0, 0, 0, 2],
44
+ ],
45
+ "marginals": [4, 6, 6, 6, 2],
46
+ "alpha": {"nominal": 0.692},
47
+ # (23 * 18 - 104) / (24 * 23 - 104)
48
+ "exact_alpha": {"nominal": Fraction(155, 224)},
49
+ }
50
+
51
+ #: Sections C and D: four observers, twelve units, seven missing values. Unit 12
52
+ #: has a single value (from observer B), so it cannot be paired.
53
+ FOUR_CODERS_MISSING: dict[str, Any] = {
54
+ "matrix": {
55
+ "A": [1, 2, 3, 3, 2, 1, 4, 1, 2, None, None, None],
56
+ "B": [1, 2, 3, 3, 2, 2, 4, 1, 2, 5, None, 3],
57
+ "C": [None, 3, 3, 3, 2, 3, 4, 2, 2, 5, 1, None],
58
+ "D": [1, 2, 3, 3, 2, 4, 4, 1, 2, 5, 1, None],
59
+ },
60
+ "values": [1, 2, 3, 4, 5],
61
+ "n_pairable": 40,
62
+ "coincidence": [
63
+ [7, 4 * THIRD, THIRD, THIRD, 0],
64
+ [4 * THIRD, 10, 4 * THIRD, THIRD, 0],
65
+ [THIRD, 4 * THIRD, 8, THIRD, 0],
66
+ [THIRD, THIRD, THIRD, 4, 0],
67
+ [0, 0, 0, 0, 3],
68
+ ],
69
+ "marginals": [9, 13, 10, 5, 3],
70
+ "alpha": {"nominal": 0.743, "ordinal": 0.815, "interval": 0.849, "ratio": 0.797},
71
+ "exact_alpha": {
72
+ "nominal": Fraction(113, 152),
73
+ "ordinal": Fraction(108577, 133160),
74
+ "interval": Fraction(951, 1120),
75
+ "ratio": Fraction(18222619, 22852465),
76
+ },
77
+ "d_o": {
78
+ "nominal": Fraction(1, 5),
79
+ "ordinal": Fraction(1891, 40),
80
+ "interval": Fraction(13, 30),
81
+ "ratio": Fraction(59357, 2646000),
82
+ },
83
+ "d_e": {
84
+ "nominal": Fraction(152, 195),
85
+ "ordinal": Fraction(3329, 13),
86
+ "interval": Fraction(112, 39),
87
+ "ratio": Fraction(4570493, 41277600),
88
+ },
89
+ # Section E: values (rows 1-5) by units (columns 1-12), n_uc, and the number
90
+ # of values in each unit, n_u.
91
+ "values_by_units": [
92
+ [3, 0, 0, 0, 0, 1, 0, 3, 0, 0, 2, 0],
93
+ [0, 3, 0, 0, 4, 1, 0, 1, 4, 0, 0, 0],
94
+ [0, 1, 4, 4, 0, 1, 0, 0, 0, 0, 0, 1],
95
+ [0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0],
96
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0],
97
+ ],
98
+ "values_per_unit": [3, 4, 4, 4, 4, 4, 4, 4, 4, 3, 2, 1],
99
+ }
100
+
101
+ #: Section D, page 6: ordinal differences delta (before squaring) for the
102
+ #: frequencies of example C with an unused 5th rank added. The paper prints the
103
+ #: upper triangle as delta squared and the lower triangle as its value; its lower
104
+ #: cell (5th, 1st) reads 992, but 32.5 squared is 1056.25, as its upper triangle
105
+ #: and formula give.
106
+ ORDINAL_DIFFERENCES: dict[str, Any] = {
107
+ "ranks": [1, 2, 3, 4, 5, 6],
108
+ "frequencies": [9, 13, 10, 5, 0, 3],
109
+ "delta": {
110
+ (1, 2): 11, (1, 3): 22.5, (1, 4): 30, (1, 5): 32.5, (1, 6): 34,
111
+ (2, 3): 11.5, (2, 4): 19, (2, 5): 21.5, (2, 6): 23,
112
+ (3, 4): 7.5, (3, 5): 10, (3, 6): 11.5,
113
+ (4, 5): 2.5, (4, 6): 4,
114
+ (5, 6): 1.5,
115
+ },
116
+ } # fmt: skip
117
+
118
+ #: Section D, page 7: ratio differences ((c - k) / (c + k))^2, printed to two decimals.
119
+ RATIO_DIFFERENCES: dict[str, Any] = {
120
+ "values": [0, 1, 2, 3, 4, 5],
121
+ "printed": {
122
+ (1, 0): 1, (2, 0): 1, (3, 0): 1, (4, 0): 1, (5, 0): 1,
123
+ (2, 1): 0.11, (3, 1): 0.25, (4, 1): 0.36, (5, 1): 0.44,
124
+ (3, 2): 0.04, (4, 2): 0.11, (5, 2): 0.18,
125
+ (4, 3): 0.02, (5, 3): 0.06,
126
+ (5, 4): 0.01,
127
+ },
128
+ } # fmt: skip
129
+
130
+ #: Section D, page 7: circular differences sin^2(180 (c - k) / U) with U = 6.
131
+ CIRCULAR_DIFFERENCES: dict[str, Any] = {
132
+ "values": [0, 1, 2, 3, 4, 5],
133
+ "circumference": 6,
134
+ "first_row": [0, 0.25, 0.75, 1, 0.75, 0.25],
135
+ }
136
+
137
+ #: Section D, page 7: bipolar differences (c - k)^2 / ((c + k - 2 c_min)(2 c_max - c - k)),
138
+ #: printed to three decimals.
139
+ BIPOLAR_DIFFERENCES: dict[str, Any] = {
140
+ "values": [-2, -1, 0, 1, 2],
141
+ "printed": {
142
+ (-1, -2): 0.143, (0, -2): 0.333, (1, -2): 0.600, (2, -2): 1.0,
143
+ (0, -1): 0.067, (1, -1): 0.250, (2, -1): 0.600,
144
+ (1, 0): 0.067, (2, 0): 0.333,
145
+ (2, 1): 0.143,
146
+ },
147
+ } # fmt: skip
148
+
149
+ EXAMPLES = {
150
+ "binary_two_coders": BINARY_TWO_CODERS,
151
+ "nominal_two_coders": NOMINAL_TWO_CODERS,
152
+ "four_coders_missing": FOUR_CODERS_MISSING,
153
+ }
@@ -64,6 +64,14 @@ def components_from_coincidences(
64
64
  i, j = sorted((order.index(a), order.index(b)))
65
65
  between = sum(n_c[order[g]] for g in range(i, j + 1))
66
66
  return float((between - (n_c[a] + n_c[b]) / 2) ** 2)
67
+ if level == "bipolar":
68
+ if a == b:
69
+ return 0.0
70
+ low, high = min(order), max(order)
71
+ return float((a - b) ** 2 / ((a + b - 2 * low) * (2 * high - a - b)))
72
+ if level.startswith("circular(") and level.endswith(")"):
73
+ circumference = float(level[len("circular(") : -1])
74
+ return math.sin(math.pi * (a - b) / circumference) ** 2
67
75
  raise ValueError(level)
68
76
 
69
77
  if n == 0:
@@ -15,7 +15,7 @@ from reliability_lab.alpha import (
15
15
  per_unit_coincidences,
16
16
  )
17
17
 
18
- LEVELS = ["nominal", "ordinal", "interval", "ratio"]
18
+ LEVELS = ["nominal", "ordinal", "interval", "ratio", "bipolar"]
19
19
 
20
20
  PUBLISHED = [
21
21
  (name, level, expected)
@@ -0,0 +1,121 @@
1
+ """The built-in bipolar and circular distances (Krippendorff 2011, section D)."""
2
+
3
+ import json
4
+ import math
5
+ from typing import Any
6
+
7
+ import numpy as np
8
+ import pytest
9
+ from helpers import human_study, long_from_matrix, random_matrix
10
+ from reference import reference_alpha
11
+
12
+ from reliability_lab import BIPOLAR, DesignError, RatingFrame, ReliabilityStudy, circular
13
+
14
+ pytestmark = pytest.mark.filterwarnings("ignore::reliability_lab.CategorySkewWarning")
15
+
16
+ HOURS = list(range(24))
17
+ #: (oracle level, measurement, values)
18
+ CASES = [
19
+ ("bipolar", "bipolar", [-2, -1, 0, 1, 2]),
20
+ ("circular(12)", circular(12), list(range(1, 13))),
21
+ ]
22
+
23
+
24
+ @pytest.mark.parametrize(("level", "measurement", "values"), CASES, ids=["bipolar", "circular"])
25
+ def test_engine_matches_reference_on_random_data(
26
+ level: str, measurement: Any, values: list[int]
27
+ ) -> None:
28
+ rng = np.random.default_rng(2029)
29
+ checked = 0
30
+ for _ in range(25):
31
+ matrix = random_matrix(
32
+ rng, int(rng.integers(2, 6)), int(rng.integers(3, 30)), values, p_missing=0.25
33
+ )
34
+ expected = reference_alpha(matrix, level, values)
35
+ if math.isnan(expected):
36
+ continue
37
+ assert human_study(matrix, measurement, values).alpha().alpha == pytest.approx(
38
+ expected, abs=1e-12
39
+ )
40
+ checked += 1
41
+ assert checked >= 20
42
+
43
+
44
+ def test_circular_needs_its_circumference() -> None:
45
+ with pytest.raises(DesignError, match=r"circular\(U\)"):
46
+ human_study({"a": [1, 2], "b": [1, 2]}, "circular", [1, 2])
47
+
48
+
49
+ @pytest.mark.parametrize("circumference", [0, -1, math.nan, math.inf, True, "24", None])
50
+ def test_circumference_must_be_a_positive_number(circumference: Any) -> None:
51
+ with pytest.raises(DesignError, match="circumference"):
52
+ circular(circumference)
53
+
54
+
55
+ def test_the_circumference_is_recorded() -> None:
56
+ matrix = {"a": [0, 6, 12, 23], "b": [1, 6, 13, 0]}
57
+ result = human_study(matrix, circular(24), HOURS).alpha()
58
+ assert result.measurement == "circular(24)"
59
+ payload = json.loads(result.to_json())
60
+ assert payload["measurement"] == "circular(24)"
61
+ assert payload["provenance"]["declarations"]["measurement"] == "circular(24)"
62
+ assert circular(6.5).name == "circular(6.5)"
63
+
64
+
65
+ def test_hours_wrap_around_midnight() -> None:
66
+ d = circular(24).matrix(HOURS, np.ones(24))
67
+ assert d[23, 0] == pytest.approx(d[0, 1])
68
+ assert d[0, 12] == pytest.approx(1.0) # opposite hours are the farthest apart
69
+ assert d.max() == pytest.approx(1.0)
70
+
71
+
72
+ def test_rotating_every_value_leaves_circular_alpha_unchanged() -> None:
73
+ rng = np.random.default_rng(7)
74
+ matrix = random_matrix(rng, 4, 30, HOURS, p_missing=0.2)
75
+ rotated = {
76
+ coder: [None if v is None else (v + 5) % 24 for v in column]
77
+ for coder, column in matrix.items()
78
+ }
79
+ a = human_study(matrix, circular(24), HOURS).alpha().alpha
80
+ b = human_study(rotated, circular(24), HOURS).alpha().alpha
81
+ assert a == pytest.approx(b, abs=1e-12)
82
+
83
+
84
+ def test_bipolar_takes_its_poles_from_the_declared_values() -> None:
85
+ narrow = BIPOLAR.matrix([-2, -1, 0, 1, 2], np.ones(5))
86
+ wide = BIPOLAR.matrix([-3, -2, -1, 0, 1, 2, 3], np.ones(7))
87
+ # -2 and 2 are the poles of the narrow scale, the farthest apart possible ...
88
+ assert narrow[0, 4] == pytest.approx(1.0)
89
+ # ... but on a -3..3 scale they are 16 / (6 * 6) apart.
90
+ assert wide[1, 5] == pytest.approx(16 / 36)
91
+ with pytest.raises(DesignError, match="numeric"):
92
+ human_study({"a": ["x", "y"], "b": ["x", "y"]}, "bipolar", ["x", "y"])
93
+
94
+
95
+ @pytest.mark.parametrize(("level", "measurement", "values"), CASES, ids=["bipolar", "circular"])
96
+ def test_bootstraps_and_sensitivity_run(level: str, measurement: Any, values: list[int]) -> None:
97
+ rng = np.random.default_rng(3)
98
+ # Six coders: with four, a two-way resample now and then draws one coder four
99
+ # times, leaving no pairs (counted and warned about, but noise in this test).
100
+ frame = long_from_matrix(random_matrix(rng, 6, 40, values, p_missing=0.1))
101
+ frame["status"] = "observed"
102
+ frame.loc[0, ["value", "status"]] = [None, "parse_error"]
103
+ ratings = RatingFrame.from_long(
104
+ frame, unit="unit", rater="coder", value="value", status="status"
105
+ )
106
+ study = ReliabilityStudy(
107
+ ratings, measurement=measurement, values=values, coders="rater_id", design="human_panel"
108
+ )
109
+ result = study.alpha()
110
+ assert set(result.estimates) == {
111
+ "as_missing",
112
+ "sensitivity_pessimistic",
113
+ "sensitivity_optimistic",
114
+ }
115
+ pessimistic, optimistic = (
116
+ result.estimates[f"sensitivity_{v}"].alpha for v in ("pessimistic", "optimistic")
117
+ )
118
+ assert pessimistic <= optimistic
119
+ for scheme in ("units", "two_way"):
120
+ boot = result.bootstrap(scheme=scheme, n_resamples=200, random_state=1) # type: ignore[arg-type]
121
+ assert boot["as_missing"].defined
@@ -0,0 +1,158 @@
1
+ """The intermediate quantities Krippendorff (2011) prints, not just its alphas:
2
+ coincidence matrices, per-unit contributions, difference tables, D_o and D_e, and
3
+ the values-by-units form of section E."""
4
+
5
+ from itertools import combinations
6
+ from typing import Any
7
+
8
+ import numpy as np
9
+ import pytest
10
+ from fixtures.krippendorff2011 import (
11
+ BIPOLAR_DIFFERENCES,
12
+ CIRCULAR_DIFFERENCES,
13
+ EXAMPLES,
14
+ FOUR_CODERS_MISSING,
15
+ ORDINAL_DIFFERENCES,
16
+ RATIO_DIFFERENCES,
17
+ )
18
+ from helpers import human_study
19
+
20
+ from reliability_lab import BIPOLAR, Distance, circular
21
+ from reliability_lab.alpha import BUILTIN_DISTANCES, INTERVAL, ORDINAL, RATIO, per_unit_coincidences
22
+
23
+ F4 = FOUR_CODERS_MISSING
24
+ LEVELS = ["nominal", "ordinal", "interval", "ratio"]
25
+ EXACT = [
26
+ (name, level, exact)
27
+ for name, example in EXAMPLES.items()
28
+ for level, exact in example["exact_alpha"].items()
29
+ ]
30
+
31
+
32
+ def _as_float(matrix: list[list[Any]]) -> np.ndarray:
33
+ return np.array([[float(x) for x in row] for row in matrix])
34
+
35
+
36
+ @pytest.mark.parametrize("name", list(EXAMPLES))
37
+ def test_coincidence_matrix_matches_the_paper(name: str) -> None:
38
+ example = EXAMPLES[name]
39
+ est = human_study(example["matrix"], "nominal", example["values"]).alpha()["as_missing"]
40
+ np.testing.assert_allclose(
41
+ est.coincidence.to_numpy(), _as_float(example["coincidence"]), atol=1e-12
42
+ )
43
+ np.testing.assert_allclose(est.coincidence.to_numpy().sum(axis=1), example["marginals"])
44
+
45
+
46
+ @pytest.mark.parametrize(("name", "level", "exact"), EXACT)
47
+ def test_alpha_matches_the_exact_value(name: str, level: str, exact: Any) -> None:
48
+ example = EXAMPLES[name]
49
+ result = human_study(example["matrix"], level, example["values"]).alpha()
50
+ assert result.alpha == pytest.approx(float(exact), abs=1e-12)
51
+ assert round(float(exact), 3) == example["alpha"][level]
52
+
53
+
54
+ @pytest.mark.parametrize("level", LEVELS)
55
+ def test_observed_and_expected_disagreement(level: str) -> None:
56
+ est = human_study(F4["matrix"], level, F4["values"]).alpha()["as_missing"]
57
+ assert est.d_o == pytest.approx(float(F4["d_o"][level]), rel=1e-12)
58
+ assert est.d_e == pytest.approx(float(F4["d_e"][level]), rel=1e-12)
59
+
60
+
61
+ def test_unit_contributions_described_in_the_text() -> None:
62
+ units, codes = [], []
63
+ for u, column in enumerate(zip(*F4["matrix"].values(), strict=True)):
64
+ for value in column:
65
+ if value is not None:
66
+ units.append(u)
67
+ codes.append(F4["values"].index(value))
68
+ per_unit, pairable = per_unit_coincidences(np.array(units), np.array(codes), 12, 5)
69
+ o = per_unit.toarray().reshape(12, 5, 5)
70
+ # Unit 1: three 1s, 6 pairs, adds 6 / (3 - 1) = 3 to o_11.
71
+ assert o[0, 0, 0] == pytest.approx(3) and o[0].sum() == pytest.approx(3)
72
+ # Unit 2: adds 2 to o_22, 1 to o_23 and 1 to o_32, accounting for its 4 values.
73
+ assert (o[1, 1, 1], o[1, 1, 2], o[1, 2, 1]) == pytest.approx((2, 1, 1))
74
+ assert o[1].sum() == pytest.approx(4)
75
+ # Unit 6: 12 mismatching pairs, each adding 1/3 to a different cell.
76
+ mismatches = o[5][~np.eye(5, dtype=bool)]
77
+ assert np.count_nonzero(mismatches) == 12
78
+ np.testing.assert_allclose(mismatches[mismatches > 0], 1 / 3)
79
+ # Unit 12: a lone value, no pairs.
80
+ assert pairable[11] == 1 and o[11].sum() == 0
81
+
82
+
83
+ @pytest.mark.parametrize("level", LEVELS)
84
+ def test_values_by_units_form(level: str) -> None:
85
+ """Section E: alpha and each unit's share of D_o from the values-by-units counts."""
86
+ est = human_study(F4["matrix"], level, F4["values"]).alpha()["as_missing"]
87
+ table = est.per_unit.set_index("unit_id").sort_index()
88
+ counts = np.array(F4["values_by_units"], dtype=float)
89
+ per_unit = np.array(F4["values_per_unit"], dtype=float)
90
+ np.testing.assert_array_equal(table["pairable_values"], per_unit)
91
+ pairable = per_unit >= 2
92
+ totals = counts[:, pairable].sum(axis=1)
93
+ np.testing.assert_array_equal(totals, F4["marginals"])
94
+ n = totals.sum()
95
+ delta2 = BUILTIN_DISTANCES[level].matrix(F4["values"], totals)
96
+ pairs = list(combinations(range(len(totals)), 2))
97
+ within = np.array(
98
+ [
99
+ sum(counts[c, u] * counts[k, u] * delta2[c, k] for c, k in pairs) / (per_unit[u] - 1)
100
+ if pairable[u]
101
+ else 0.0
102
+ for u in range(counts.shape[1])
103
+ ]
104
+ )
105
+ # A unit's share of D_o counts both orders of each pair.
106
+ np.testing.assert_allclose(table["d_o_contribution"], 2 * within / n, atol=1e-12)
107
+ between = sum(totals[c] * totals[k] * delta2[c, k] for c, k in pairs)
108
+ alpha = 1 - (n - 1) * within.sum() / between
109
+ assert alpha == pytest.approx(float(F4["exact_alpha"][level]), abs=1e-12)
110
+
111
+
112
+ def test_ordinal_differences_table() -> None:
113
+ table = ORDINAL_DIFFERENCES
114
+ delta2 = ORDINAL.matrix(table["ranks"], table["frequencies"])
115
+ for (c, k), delta in table["delta"].items():
116
+ assert delta2[c - 1, k - 1] == pytest.approx(delta**2)
117
+ assert delta2[k - 1, c - 1] == pytest.approx(delta**2)
118
+
119
+
120
+ def test_an_unused_rank_does_not_change_ordinal_alpha() -> None:
121
+ shifted = {coder: [6 if v == 5 else v for v in col] for coder, col in F4["matrix"].items()}
122
+ alpha = human_study(shifted, "ordinal", [1, 2, 3, 4, 5, 6]).alpha().alpha
123
+ assert alpha == pytest.approx(float(F4["exact_alpha"]["ordinal"]), abs=1e-12)
124
+
125
+
126
+ def test_standardised_ordinal_differences_give_the_same_alpha() -> None:
127
+ def standardised(values: Any, marginals: Any) -> Any:
128
+ d = ORDINAL.fn(values, marginals)
129
+ return d / d.max()
130
+
131
+ distance = Distance("ordinal_standardised", standardised, depends_on_marginals=True)
132
+ alpha = human_study(F4["matrix"], distance, F4["values"]).alpha().alpha
133
+ assert alpha == pytest.approx(float(F4["exact_alpha"]["ordinal"]), abs=1e-12)
134
+
135
+
136
+ def test_interval_and_ratio_tables() -> None:
137
+ values = np.arange(-1, 5)
138
+ expected = (values[:, None] - values[None, :]) ** 2
139
+ np.testing.assert_array_equal(INTERVAL.matrix(list(values), np.ones(6)), expected)
140
+ ratio = RATIO.matrix(RATIO_DIFFERENCES["values"], np.ones(6))
141
+ for (c, k), printed in RATIO_DIFFERENCES["printed"].items():
142
+ assert round(ratio[c, k], 2) == printed
143
+ assert ratio[k, c] == ratio[c, k]
144
+
145
+
146
+ def test_circular_and_bipolar_tables() -> None:
147
+ d = circular(CIRCULAR_DIFFERENCES["circumference"]).matrix(
148
+ CIRCULAR_DIFFERENCES["values"], np.ones(6)
149
+ )
150
+ np.testing.assert_allclose(d[0], CIRCULAR_DIFFERENCES["first_row"], atol=1e-12)
151
+ # Every row is the first row rotated: only the distance around the circle counts.
152
+ for c in range(6):
153
+ np.testing.assert_allclose(d[c], np.roll(d[0], c), atol=1e-12)
154
+
155
+ values = BIPOLAR_DIFFERENCES["values"]
156
+ d = BIPOLAR.matrix(values, np.ones(5))
157
+ for (c, k), printed in BIPOLAR_DIFFERENCES["printed"].items():
158
+ assert round(d[values.index(c), values.index(k)], 3) == printed
@@ -11,7 +11,7 @@ from reference import reference_alpha
11
11
 
12
12
  from reliability_lab import RatingFrame, ReliabilityStudy, UndefinedAlphaError
13
13
 
14
- LEVELS = ["nominal", "ordinal", "interval", "ratio"]
14
+ LEVELS = ["nominal", "ordinal", "interval", "ratio", "bipolar"]
15
15
 
16
16
  # Random data often has one dominant category; the warning is expected here.
17
17
  pytestmark = pytest.mark.filterwarnings("ignore::reliability_lab.CategorySkewWarning")
@@ -1 +0,0 @@
1
- __version__ = "0.1.3"
@@ -1,52 +0,0 @@
1
- """Worked examples from Krippendorff (2011), *Computing Krippendorff's alpha-reliability*.
2
-
3
- Matrices are coders x units; ``None`` marks a missing rating. Data and expected
4
- values are transcribed from the paper (checked against its text on 26 Oct 2026).
5
- The paper prints alpha to three decimals. The ratio value for example C did not
6
- extract cleanly from the PDF, so it is also covered by the parity tests against
7
- the ``krippendorff`` package.
8
- """
9
-
10
- from typing import Any
11
-
12
- #: Section A: binary data, two observers, no missing data.
13
- BINARY_TWO_CODERS: dict[str, Any] = {
14
- "matrix": {
15
- "Meg": [0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
16
- "Owen": [1, 1, 1, 0, 0, 1, 0, 0, 0, 0],
17
- },
18
- "values": [0, 1],
19
- "n_pairable": 20,
20
- "alpha": {"nominal": 0.095},
21
- }
22
-
23
- #: Section B: nominal data, two observers, no missing data.
24
- NOMINAL_TWO_CODERS: dict[str, Any] = {
25
- "matrix": {
26
- "Ben": list("aabbdcccedda"),
27
- "Gerry": list("babbbccceddd"),
28
- },
29
- "values": ["a", "b", "c", "d", "e"],
30
- "n_pairable": 24,
31
- "alpha": {"nominal": 0.692},
32
- }
33
-
34
- #: Sections C and D: four observers, twelve units, seven missing values. Unit 12
35
- #: has a single value (from observer B), so it cannot be paired.
36
- FOUR_CODERS_MISSING: dict[str, Any] = {
37
- "matrix": {
38
- "A": [1, 2, 3, 3, 2, 1, 4, 1, 2, None, None, None],
39
- "B": [1, 2, 3, 3, 2, 2, 4, 1, 2, 5, None, 3],
40
- "C": [None, 3, 3, 3, 2, 3, 4, 2, 2, 5, 1, None],
41
- "D": [1, 2, 3, 3, 2, 4, 4, 1, 2, 5, 1, None],
42
- },
43
- "values": [1, 2, 3, 4, 5],
44
- "n_pairable": 40,
45
- "alpha": {"nominal": 0.743, "ordinal": 0.815, "interval": 0.849, "ratio": 0.797},
46
- }
47
-
48
- EXAMPLES = {
49
- "binary_two_coders": BINARY_TWO_CODERS,
50
- "nominal_two_coders": NOMINAL_TWO_CODERS,
51
- "four_coders_missing": FOUR_CODERS_MISSING,
52
- }
File without changes
File without changes