reliability-lab 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/CHANGELOG.md +21 -0
  2. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/PKG-INFO +7 -6
  3. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/README.md +5 -5
  4. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/pyproject.toml +1 -0
  5. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/__init__.py +3 -1
  6. reliability_lab-0.1.4/src/reliability_lab/_version.py +1 -0
  7. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/alpha.py +47 -1
  8. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/inference.py +117 -97
  9. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/study.py +10 -2
  10. reliability_lab-0.1.4/tests/fixtures/krippendorff2011.py +153 -0
  11. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/reference.py +8 -0
  12. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_alpha.py +1 -1
  13. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_bca.py +83 -20
  14. reliability_lab-0.1.4/tests/test_circular_bipolar.py +121 -0
  15. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_compare.py +51 -0
  16. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_inference_units.py +18 -0
  17. reliability_lab-0.1.4/tests/test_krippendorff2011.py +158 -0
  18. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_properties.py +1 -1
  19. reliability_lab-0.1.2/src/reliability_lab/_version.py +0 -1
  20. reliability_lab-0.1.2/tests/fixtures/krippendorff2011.py +0 -52
  21. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/.gitattributes +0 -0
  22. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/.github/workflows/ci.yml +0 -0
  23. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/.github/workflows/release.yml +0 -0
  24. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/.gitignore +0 -0
  25. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/.python-version +0 -0
  26. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/LICENSE +0 -0
  27. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/design/framing-v2.md +0 -0
  28. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/bootstrap-coverage.md +0 -0
  29. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-dark.png +0 -0
  30. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_factor-light.png +0 -0
  31. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-dark.png +0 -0
  32. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_n-light.png +0 -0
  33. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-dark.png +0 -0
  34. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/guidance/figures/coverage_by_spread-light.png +0 -0
  35. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/docs/releasing.md +0 -0
  36. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/examples/llm_judge.py +0 -0
  37. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/examples/quickstart.py +0 -0
  38. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/examples/splits.py +0 -0
  39. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/__init__.py +0 -0
  40. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/coders.py +0 -0
  41. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/coverage.py +0 -0
  42. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/grid.py +0 -0
  43. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/report.py +0 -0
  44. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/results/coders/coverage.csv +0 -0
  45. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/results/coders/run_info.json +0 -0
  46. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/results/coverage.csv +0 -0
  47. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/results/run_info.json +0 -0
  48. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/run_coverage.py +0 -0
  49. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/simulations/templates/bootstrap-coverage.md +0 -0
  50. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/data.py +0 -0
  51. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/design.py +0 -0
  52. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/exceptions.py +0 -0
  53. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/llm.py +0 -0
  54. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/py.typed +0 -0
  55. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/report.py +0 -0
  56. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/src/reliability_lab/split.py +0 -0
  57. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/fixtures/__init__.py +0 -0
  58. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/fixtures/llm_logs.py +0 -0
  59. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/helpers.py +0 -0
  60. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/naive_bootstrap.py +0 -0
  61. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_coder_jackknife.py +0 -0
  62. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_coder_resampling.py +0 -0
  63. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_data.py +0 -0
  64. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_design.py +0 -0
  65. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_kalpha.py +0 -0
  66. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_llm_end_to_end.py +0 -0
  67. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_parity.py +0 -0
  68. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_report.py +0 -0
  69. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_report_io.py +0 -0
  70. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_retries.py +0 -0
  71. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_simulations.py +0 -0
  72. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_smoke.py +0 -0
  73. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_split.py +0 -0
  74. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_status_policies.py +0 -0
  75. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/tests/test_trace.py +0 -0
  76. {reliability_lab-0.1.2 → reliability_lab-0.1.4}/uv.lock +0 -0
@@ -2,6 +2,27 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ## 0.1.4
6
+
7
+ All six of Krippendorff's difference functions.
8
+
9
+ - Two more built-in distances from Krippendorff (2011):
10
+ - `measurement="bipolar"` is for scales with two poles, such as −3…+3. Its poles are the smallest and largest declared values.
11
+ - `measurement=circular(U)` is for values on a circle of circumference U: hours (24), degrees (360), months (12). U must be declared, so the string `"circular"` is refused.
12
+
13
+ ## 0.1.3
14
+
15
+ Faster bootstrap.
16
+
17
+ - Every scheme is faster. With 2,000 resamples of 5,000 units and 5 coders (ordinal, 3 estimates):
18
+ - units, percentile: 1.4 s → 0.5 s;
19
+ - units, BCa: 6.4 s → 0.5 s, and `compare(method="bca")`: 18 s → 0.5 s;
20
+ - kalpha: 5.1 s → 2.8 s;
21
+ - coders: 8.2 s → 1.4 s, and two_way: 10 s → 2.1 s.
22
+ - BCa intervals no longer go through `scipy.stats.bootstrap`. The jackknife leaves each cluster out in closed form, instead of recomputing alpha from k × k resample weights, so BCa now runs on large studies: 20,000 units take 3.7 s, where they previously ran out of memory. The resamples and the interval are still the ones `scipy.stats.bootstrap` gives, and the tests check this.
23
+ - The random draws are unchanged: a given `random_state` gives the same intervals as in 0.1.2, up to floating-point rounding.
24
+ - The README links to the docs and examples on GitHub, so the links work on PyPI. The project links on PyPI include the changelog.
25
+
5
26
  ## 0.1.2
6
27
 
7
28
  Generalising over coders.
@@ -1,9 +1,10 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: reliability-lab
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
5
5
  Project-URL: Repository, https://github.com/Sathapana/reliability-lab
6
6
  Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
7
+ Project-URL: Changelog, https://github.com/Sathapana/reliability-lab/blob/main/CHANGELOG.md
7
8
  Author: Sathapana Chawananon
8
9
  License-Expression: BSD-3-Clause
9
10
  License-File: LICENSE
@@ -30,7 +31,7 @@ Description-Content-Type: text/markdown
30
31
 
31
32
  # reliability-lab
32
33
 
33
- **Status: alpha (0.1.2).** The API may still change between releases.
34
+ **Status: alpha (0.1.4).** The API may still change between releases.
34
35
 
35
36
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
36
37
 
@@ -45,7 +46,7 @@ reliability-lab is a study-design library for measuring the reliability of LLM a
45
46
  - versioned JSON and Parquet reports with provenance;
46
47
  - LLM judge workflows: a trace sidecar of judge configurations, declared retry handling, and evidence specific to each design (run agreement, order swaps and position bias).
47
48
 
48
- Mixed human-LLM panels are next on the roadmap ([design document](docs/design/framing-v2.md)).
49
+ Mixed human-LLM panels are next on the roadmap ([design document](https://github.com/Sathapana/reliability-lab/blob/main/docs/design/framing-v2.md)).
49
50
 
50
51
  > **Reliability is not validity.** Coders can agree consistently and still be wrong in the same way, for example because of a flawed rubric or a bias they share. When results come from different designs (repeatability, inter-model agreement, human panels), they are reported as different quantities and are never pooled.
51
52
 
@@ -111,7 +112,7 @@ print(jk.summary()) # interval, standard error, and the most influential coders
111
112
 
112
113
  When coders are resampled, the copies of one coder are never paired with each other, since a coder always agrees with itself. These schemes need at least 3 coders and give percentile intervals.
113
114
 
114
- Which scheme to trust, and when, is measured by simulation in the [bootstrap coverage guidance](docs/guidance/bootstrap-coverage.md).
115
+ Which scheme to trust, and when, is measured by simulation in the [bootstrap coverage guidance](https://github.com/Sathapana/reliability-lab/blob/main/docs/guidance/bootstrap-coverage.md).
115
116
 
116
117
  To compare two configurations of a judge, for example two prompts, select each one's rows. Both must cover the same units. Every replicate resamples the same units for both, and the interval is on the difference:
117
118
 
@@ -150,7 +151,7 @@ dev2, test2 = later_study.apply_manifest("split.json") # same units, new judge
150
151
  test.manifest.assert_no_leakage(few_shot_units) # raises LeakageError
151
152
  ```
152
153
 
153
- See [`examples/splits.py`](examples/splits.py) for a full run. Releases are described in [docs/releasing.md](docs/releasing.md).
154
+ See [`examples/splits.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/splits.py) for a full run. Releases are described in [docs/releasing.md](https://github.com/Sathapana/reliability-lab/blob/main/docs/releasing.md).
154
155
 
155
156
  ## LLM judge workflows
156
157
 
@@ -190,7 +191,7 @@ result.trace_findings # confounds and caveats from the trace
190
191
 
191
192
  The single-configuration designs (repeatability, prompt robustness, order robustness) can hold several configurations to compare, such as an old and a new judge. Each configuration is checked on its own. `alpha()` refuses to pool them; `study.compare(a={"condition_id": "old"}, b={"condition_id": "new"})` reports each one's alpha and diagnostics, with a paired interval on the difference.
192
193
 
193
- Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](examples/llm_judge.py).
194
+ Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
194
195
 
195
196
  ## Licence
196
197
 
@@ -1,6 +1,6 @@
1
1
  # reliability-lab
2
2
 
3
- **Status: alpha (0.1.2).** The API may still change between releases.
3
+ **Status: alpha (0.1.4).** The API may still change between releases.
4
4
 
5
5
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
6
6
 
@@ -15,7 +15,7 @@ reliability-lab is a study-design library for measuring the reliability of LLM a
15
15
  - versioned JSON and Parquet reports with provenance;
16
16
  - LLM judge workflows: a trace sidecar of judge configurations, declared retry handling, and evidence specific to each design (run agreement, order swaps and position bias).
17
17
 
18
- Mixed human-LLM panels are next on the roadmap ([design document](docs/design/framing-v2.md)).
18
+ Mixed human-LLM panels are next on the roadmap ([design document](https://github.com/Sathapana/reliability-lab/blob/main/docs/design/framing-v2.md)).
19
19
 
20
20
  > **Reliability is not validity.** Coders can agree consistently and still be wrong in the same way, for example because of a flawed rubric or a bias they share. When results come from different designs (repeatability, inter-model agreement, human panels), they are reported as different quantities and are never pooled.
21
21
 
@@ -81,7 +81,7 @@ print(jk.summary()) # interval, standard error, and the most influential coders
81
81
 
82
82
  When coders are resampled, the copies of one coder are never paired with each other, since a coder always agrees with itself. These schemes need at least 3 coders and give percentile intervals.
83
83
 
84
- Which scheme to trust, and when, is measured by simulation in the [bootstrap coverage guidance](docs/guidance/bootstrap-coverage.md).
84
+ Which scheme to trust, and when, is measured by simulation in the [bootstrap coverage guidance](https://github.com/Sathapana/reliability-lab/blob/main/docs/guidance/bootstrap-coverage.md).
85
85
 
86
86
  To compare two configurations of a judge, for example two prompts, select each one's rows. Both must cover the same units. Every replicate resamples the same units for both, and the interval is on the difference:
87
87
 
@@ -120,7 +120,7 @@ dev2, test2 = later_study.apply_manifest("split.json") # same units, new judge
120
120
  test.manifest.assert_no_leakage(few_shot_units) # raises LeakageError
121
121
  ```
122
122
 
123
- See [`examples/splits.py`](examples/splits.py) for a full run. Releases are described in [docs/releasing.md](docs/releasing.md).
123
+ See [`examples/splits.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/splits.py) for a full run. Releases are described in [docs/releasing.md](https://github.com/Sathapana/reliability-lab/blob/main/docs/releasing.md).
124
124
 
125
125
  ## LLM judge workflows
126
126
 
@@ -160,7 +160,7 @@ result.trace_findings # confounds and caveats from the trace
160
160
 
161
161
  The single-configuration designs (repeatability, prompt robustness, order robustness) can hold several configurations to compare, such as an old and a new judge. Each configuration is checked on its own. `alpha()` refuses to pool them; `study.compare(a={"condition_id": "old"}, b={"condition_id": "new"})` reports each one's alpha and diagnostics, with a paired interval on the difference.
162
162
 
163
- Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](examples/llm_judge.py).
163
+ Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
164
164
 
165
165
  ## Licence
166
166
 
@@ -44,6 +44,7 @@ parquet = ["pyarrow>=15"]
44
44
  [project.urls]
45
45
  Repository = "https://github.com/Sathapana/reliability-lab"
46
46
  Issues = "https://github.com/Sathapana/reliability-lab/issues"
47
+ Changelog = "https://github.com/Sathapana/reliability-lab/blob/main/CHANGELOG.md"
47
48
 
48
49
  [dependency-groups]
49
50
  dev = [
@@ -1,7 +1,7 @@
1
1
  """reliability-lab: study design for the reliability of LLM and human judgments."""
2
2
 
3
3
  from reliability_lab._version import __version__
4
- from reliability_lab.alpha import INTERVAL, NOMINAL, ORDINAL, RATIO, Distance
4
+ from reliability_lab.alpha import BIPOLAR, INTERVAL, NOMINAL, ORDINAL, RATIO, Distance, circular
5
5
  from reliability_lab.data import RatingFrame, Status
6
6
  from reliability_lab.design import CoderSpec, Design
7
7
  from reliability_lab.exceptions import (
@@ -33,6 +33,7 @@ from reliability_lab.split import SplitManifest
33
33
  from reliability_lab.study import Fold, ReliabilityStudy
34
34
 
35
35
  __all__ = [
36
+ "BIPOLAR",
36
37
  "INTERVAL",
37
38
  "NOMINAL",
38
39
  "ORDINAL",
@@ -67,4 +68,5 @@ __all__ = [
67
68
  "TraceWarning",
68
69
  "UndefinedAlphaError",
69
70
  "__version__",
71
+ "circular",
70
72
  ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.4"
@@ -13,11 +13,16 @@ on the distance. Precomputing them once therefore serves every distance and ever
13
13
  bootstrap replicate. Only distances that depend on the marginals (ordinal) must be
14
14
  rebuilt from each replicate's ``n_c``.
15
15
 
16
+ The built-in distances (nominal, ordinal, interval, ratio, bipolar and circular)
17
+ are the difference functions of that paper.
18
+
16
19
  This module works on integer-coded numpy arrays only; ``study.py`` does the coding.
17
20
  """
18
21
 
19
22
  from __future__ import annotations
20
23
 
24
+ import math
25
+ import numbers
21
26
  from collections.abc import Callable, Sequence
22
27
  from dataclasses import dataclass
23
28
  from typing import Any
@@ -124,12 +129,53 @@ def _ratio(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
124
129
  return np.asarray(np.where(total == 0, 0.0, (diff / safe) ** 2), dtype=float)
125
130
 
126
131
 
132
+ def _bipolar(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
133
+ # delta² = (c - k)² / ((c + k - 2 c_min)(2 c_max - c - k)), with c_min and c_max
134
+ # the ends of the declared scale. The denominator is zero only where c = k at an
135
+ # end, where the distance is zero.
136
+ v = values.astype(float)
137
+ c, k = v[:, None], v[None, :]
138
+ denominator = (c + k - 2 * v.min()) * (2 * v.max() - c - k)
139
+ safe = np.where(denominator == 0, 1.0, denominator)
140
+ return np.asarray(np.where(c == k, 0.0, (c - k) ** 2 / safe), dtype=float)
141
+
142
+
127
143
  NOMINAL = Distance("nominal", _nominal)
128
144
  ORDINAL = Distance("ordinal", _ordinal, depends_on_marginals=True)
129
145
  INTERVAL = Distance("interval", _interval, numeric=True)
130
146
  RATIO = Distance("ratio", _ratio, numeric=True)
147
+ BIPOLAR = Distance("bipolar", _bipolar, numeric=True)
148
+
149
+ BUILTIN_DISTANCES: dict[str, Distance] = {
150
+ d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO, BIPOLAR)
151
+ }
152
+
153
+
154
+ def circular(circumference: float) -> Distance:
155
+ """The circular distance sin²(π (c - k) / U) for values on a circle.
156
+
157
+ ``circumference`` is U, the length of one full turn in the values' units: 24
158
+ for hours of the day, 360 for degrees, 12 for months, 7 for days of the week.
159
+ The values alone cannot fix it (the last hour of the day may never be rated),
160
+ so it is always declared.
161
+ """
162
+ if (
163
+ isinstance(circumference, bool)
164
+ or not isinstance(circumference, numbers.Real)
165
+ or not math.isfinite(circumference)
166
+ or circumference <= 0
167
+ ):
168
+ raise DesignError(
169
+ f"the circumference must be a finite number above 0, got {circumference!r}"
170
+ )
171
+ u = float(circumference)
172
+ label = str(int(u)) if u.is_integer() else repr(u)
173
+
174
+ def fn(values: npt.NDArray[Any], marginals: FloatArray) -> FloatArray:
175
+ v = values.astype(float)
176
+ return np.asarray(np.sin(np.pi * (v[:, None] - v[None, :]) / u) ** 2, dtype=float)
131
177
 
132
- BUILTIN_DISTANCES: dict[str, Distance] = {d.name: d for d in (NOMINAL, ORDINAL, INTERVAL, RATIO)}
178
+ return Distance(f"circular({label})", fn, numeric=True)
133
179
 
134
180
 
135
181
  def check_domain_size(n_values: int) -> str | None:
@@ -23,15 +23,14 @@ configurations, so the interval falls on the difference alpha_b - alpha_a.
23
23
 
24
24
  from __future__ import annotations
25
25
 
26
- import warnings
27
- from collections.abc import Callable, Mapping
26
+ from collections.abc import Mapping
28
27
  from dataclasses import dataclass
29
28
  from typing import Any
30
29
 
31
30
  import numpy as np
32
31
  import numpy.typing as npt
33
32
  import pandas as pd
34
- from scipy import sparse, stats
33
+ from scipy import sparse, special, stats
35
34
 
36
35
  from reliability_lab.alpha import ORDINAL, Distance, compute_alpha, per_unit_coincidences
37
36
  from reliability_lab.data import preview
@@ -218,17 +217,41 @@ def replicate_alphas(
218
217
  distance: Distance,
219
218
  ) -> FloatArray:
220
219
  """Alpha for each row of cluster weights; NaN where alpha is undefined."""
221
- w = np.atleast_2d(np.asarray(weights, dtype=float))
220
+ w = np.atleast_2d(np.asarray(weights))
222
221
  n_values = len(values)
222
+ n_flat = n_values * n_values
223
+ # A dense product is one BLAS call; large value domains stay sparse.
224
+ dense = clusters.shape[0] * n_flat <= BATCH_FLOATS
225
+ contributions = clusters.toarray() if dense else sparse.csr_array(clusters.T)
223
226
  out = np.empty(len(w))
224
- batch = max(1, BATCH_FLOATS // max(n_values * n_values, 1))
227
+ batch = max(1, BATCH_FLOATS // max(n_flat, clusters.shape[0], 1))
225
228
  for start in range(0, len(w), batch):
226
- wb = w[start : start + batch]
227
- o = np.asarray(clusters.T @ wb.T, dtype=float).T.reshape(len(wb), n_values, n_values)
229
+ wb = np.asarray(w[start : start + batch], dtype=float)
230
+ if dense:
231
+ flat = wb @ contributions
232
+ else:
233
+ flat = np.asarray(contributions @ np.asfortranarray(wb.T)).T
234
+ o = flat.reshape(len(wb), n_values, n_values)
228
235
  out[start : start + len(wb)] = alphas_from_coincidences(o, values, distance)
229
236
  return out
230
237
 
231
238
 
239
+ def jackknife_alphas(
240
+ clusters: sparse.csr_array, values: tuple[Any, ...], distance: Distance
241
+ ) -> FloatArray:
242
+ """Alpha without each cluster in turn: the coincidences less that cluster's."""
243
+ n_values = len(values)
244
+ n_flat = n_values * n_values
245
+ total = np.asarray(clusters.sum(axis=0), dtype=float).ravel()
246
+ out = np.empty(clusters.shape[0])
247
+ batch = max(1, BATCH_FLOATS // max(n_flat, 1))
248
+ for start in range(0, clusters.shape[0], batch):
249
+ part = clusters[start : start + batch].toarray()
250
+ o = (total - part).reshape(len(part), n_values, n_values)
251
+ out[start : start + len(part)] = alphas_from_coincidences(o, values, distance)
252
+ return out
253
+
254
+
232
255
  def alphas_from_coincidences(
233
256
  o: FloatArray, values: tuple[Any, ...], distance: Distance
234
257
  ) -> FloatArray:
@@ -285,25 +308,26 @@ def two_way_alphas(
285
308
  ``(n_u n_u' - diag n_u) / (m_u - 1)``.
286
309
  """
287
310
  v = np.atleast_2d(np.asarray(coder_weights, dtype=float))
288
- n_units, _, n_values = onehot.shape
289
- w = (
290
- np.ones((len(v), n_units))
291
- if unit_weights is None
292
- else np.atleast_2d(np.asarray(unit_weights, dtype=float))
293
- )
311
+ n_units, n_coders, n_values = onehot.shape
312
+ w = None if unit_weights is None else np.atleast_2d(np.asarray(unit_weights))
313
+ # Every sum over coders is a matrix product with the coder weights.
314
+ by_coder = onehot.transpose(1, 0, 2).reshape(n_coders, n_units * n_values)
315
+ rated = onehot.sum(axis=2).T
294
316
  out = np.empty(len(v))
295
317
  per_replicate = max(n_units * n_values * 2, n_values * n_values, 1)
296
318
  batch = max(1, BATCH_FLOATS // per_replicate)
297
319
  for start in range(0, len(v), batch):
298
- vb, wb = v[start : start + batch], w[start : start + batch]
299
- counts = np.einsum("ujc,rj->ruc", onehot, vb)
300
- squares = np.einsum("ujc,rj->ruc", onehot, vb * vb)
301
- m = counts.sum(axis=2)
302
- pairs = m * m - squares.sum(axis=2)
320
+ vb = v[start : start + batch]
321
+ v2 = vb * vb
322
+ wb = 1.0 if w is None else np.asarray(w[start : start + batch], dtype=float)
323
+ counts = (vb @ by_coder).reshape(len(vb), n_units, n_values)
324
+ squares = (v2 @ by_coder).reshape(len(vb), n_units, n_values)
325
+ m = vb @ rated
326
+ pairs = m * m - v2 @ rated
303
327
  with np.errstate(divide="ignore", invalid="ignore"):
304
328
  scale = np.where(pairs > 0, wb * m / pairs, 0.0)
305
- o = np.einsum("ru,ruc,rud->rcd", scale, counts, counts)
306
- diagonal = np.einsum("ru,ruc->rc", scale, squares)
329
+ o = (counts * scale[:, :, None]).transpose(0, 2, 1) @ counts
330
+ diagonal = (scale[:, None, :] @ squares)[:, 0, :]
307
331
  o[:, np.arange(n_values), np.arange(n_values)] -= diagonal
308
332
  out[start : start + len(vb)] = alphas_from_coincidences(o, values, distance)
309
333
  return out
@@ -389,12 +413,14 @@ def kalpha_replicates(pool: KalphaPool, draws: IntArray, redraws: IntArray) -> F
389
413
  ``redraws`` (replicates x len(second_pick)) replaces a unit's second draw when
390
414
  it repeats the first.
391
415
  """
392
- d = np.array(draws, dtype=np.intp)
393
- if len(pool.second_pick):
394
- first = d[:, pool.second_pick - 1]
395
- second = d[:, pool.second_pick]
396
- d[:, pool.second_pick] = np.where(first == second, redraws, second)
397
- alpha = 1.0 - (pool.errors[d] * pool.weights).sum(axis=1)
416
+ d = np.asarray(draws, dtype=np.intp)
417
+ sp = pool.second_pick
418
+ alpha = 1.0 - np.take(pool.errors, d) @ pool.weights
419
+ if len(sp):
420
+ # Repeats are rare, so their redraws correct the sums afterwards.
421
+ rows, cols = np.nonzero(d[:, sp - 1] == d[:, sp])
422
+ redrawn = pool.errors[np.asarray(redraws)[rows, cols]] - pool.errors[d[rows, sp[cols]]]
423
+ np.subtract.at(alpha, rows, redrawn * pool.weights[sp[cols]])
398
424
  return np.asarray(np.maximum(alpha, -1.0), dtype=float)
399
425
 
400
426
 
@@ -452,37 +478,34 @@ def percentile_interval(
452
478
  def bca_interval(
453
479
  name: str,
454
480
  point: float,
455
- statistic: Callable[[IntArray], FloatArray],
456
- n_clusters: int,
457
- n_resamples: int,
481
+ distribution: FloatArray,
482
+ theta_hat: float,
483
+ jackknife: FloatArray,
458
484
  confidence: float,
459
- seed: int,
460
485
  alpha_min: float | None = None,
461
486
  ) -> Interval:
462
- """BCa interval through ``scipy.stats.bootstrap`` over resampled cluster indices."""
463
-
464
- def vectorised(idx: IntArray, axis: int = -1) -> FloatArray:
465
- return statistic(np.moveaxis(np.asarray(idx, dtype=np.intp), axis, -1))
466
-
467
- # scipy's own warning about a degenerate BCa is replaced by this library's
468
- # BootstrapWarning, which names the estimate and the fallback.
469
- with np.errstate(divide="ignore", invalid="ignore"), warnings.catch_warnings():
470
- warnings.simplefilter("ignore", stats.DegenerateDataWarning)
471
- res = stats.bootstrap(
472
- (np.arange(n_clusters),),
473
- vectorised,
474
- n_resamples=n_resamples,
475
- confidence_level=confidence,
476
- method="BCa",
477
- vectorized=True,
478
- rng=np.random.default_rng(seed),
479
- )
480
- distribution = np.asarray(res.bootstrap_distribution, dtype=float)
487
+ """BCa interval (Efron and Tibshirani 1993, 14.3 and 15.4), as ``scipy.stats.bootstrap`` does.
488
+
489
+ ``theta_hat`` is the statistic on the full sample, computed like the replicates,
490
+ and ``jackknife`` the statistic without each cluster in turn.
491
+ """
492
+ n_resamples = len(distribution)
481
493
  failed = np.isnan(distribution)
482
- low, high = float(res.confidence_interval.low), float(res.confidence_interval.high)
483
- reason = None
484
- if not (np.isfinite(low) and np.isfinite(high)):
485
- reason = "the BCa interval is undefined (undefined replicates or a degenerate jackknife)"
494
+ z_alpha = float(special.ndtri((1.0 - confidence) / 2.0))
495
+ with np.errstate(divide="ignore", invalid="ignore"):
496
+ below_hat = (distribution < theta_hat).sum() + (distribution <= theta_hat).sum()
497
+ z0 = special.ndtri(below_hat / (2 * n_resamples))
498
+ u = jackknife.mean() - jackknife
499
+ a_hat = (u**3).sum() / (6.0 * (u**2).sum() ** 1.5)
500
+ num = z0 + np.array([z_alpha, -z_alpha])
501
+ levels = special.ndtr(z0 + num / (1.0 - a_hat * num))
502
+ low = high = np.nan
503
+ reason: str | None = (
504
+ "the BCa interval is undefined (undefined replicates or a degenerate jackknife)"
505
+ )
506
+ if not failed.any() and np.isfinite(levels).all():
507
+ low, high = (float(q) for q in np.quantile(distribution, levels))
508
+ reason = None
486
509
  defined = distribution[~failed]
487
510
  below = float((defined < alpha_min).mean()) if alpha_min is not None and len(defined) else None
488
511
  return Interval(
@@ -490,6 +513,30 @@ def bca_interval(
490
513
  )
491
514
 
492
515
 
516
+ def bca_weights(seed: int, n_clusters: int, n_resamples: int) -> npt.NDArray[np.int64]:
517
+ """Cluster weights of the BCa resamples (n x clusters).
518
+
519
+ The indices are the ones ``scipy.stats.bootstrap`` draws from the same seed,
520
+ drawn in batches (which leaves the stream unchanged) and counted.
521
+ """
522
+ rng = np.random.default_rng(seed)
523
+ batch = max(1, BATCH_FLOATS // max(n_clusters, 1))
524
+ parts = []
525
+ for start in range(0, n_resamples, batch):
526
+ size = min(batch, n_resamples - start)
527
+ idx = rng.integers(0, n_clusters, (size, n_clusters))
528
+ parts.append(weights_from_indices(idx, n_clusters))
529
+ return np.concatenate(parts)
530
+
531
+
532
+ def bca_parts(
533
+ clusters: sparse.csr_array, values: tuple[Any, ...], distance: Distance
534
+ ) -> tuple[float, FloatArray]:
535
+ """The full-sample statistic and the leave-one-cluster-out jackknife for BCa."""
536
+ theta_hat = replicate_alphas(clusters, np.ones(clusters.shape[0]), values, distance)[0]
537
+ return float(theta_hat), jackknife_alphas(clusters, values, distance)
538
+
539
+
493
540
  def weights_from_indices(idx: IntArray, n_clusters: int) -> npt.NDArray[np.int64]:
494
541
  """Count cluster indices along the last axis: (..., k) -> (prod(...), n_clusters)."""
495
542
  flat = np.asarray(idx, dtype=np.intp).reshape(-1, np.shape(idx)[-1])
@@ -498,17 +545,6 @@ def weights_from_indices(idx: IntArray, n_clusters: int) -> npt.NDArray[np.int64
498
545
  return counts.reshape(len(flat), n_clusters)
499
546
 
500
547
 
501
- def _alpha_statistic(
502
- clusters: sparse.csr_array, n_clusters: int, values: tuple[Any, ...], distance: Distance
503
- ) -> Callable[[IntArray], FloatArray]:
504
- def statistic(idx: IntArray) -> FloatArray:
505
- lead = np.shape(idx)[:-1]
506
- w = weights_from_indices(idx, n_clusters)
507
- return replicate_alphas(clusters, w, values, distance).reshape(lead)
508
-
509
- return statistic
510
-
511
-
512
548
  # -- entry points ---------------------------------------------------------------------
513
549
 
514
550
 
@@ -624,35 +660,27 @@ def bootstrap(
624
660
  recorded.append(emit(BootstrapWarning, level_warning, stacklevel=3))
625
661
  if method == "bca" and strata:
626
662
  raise DesignError(
627
- "BCa intervals cannot resample within strata (scipy.stats.bootstrap has no "
628
- "strata); use method='percentile'"
663
+ "BCa intervals cannot resample within strata; use method='percentile'"
629
664
  )
630
665
  plan = cluster_plan(sample, level_used, strata)
631
666
  n_clusters = plan.n_clusters
632
667
  weights = (
633
668
  draw_weights(np.random.default_rng(seed), plan, n_resamples)
634
669
  if method == "percentile"
635
- else None
670
+ else bca_weights(seed, n_clusters, n_resamples)
636
671
  )
637
672
  for name in names:
638
673
  values = sample.inputs[name].values
639
674
  clusters = cluster_coincidences(sample, name, plan)
640
- if weights is not None:
641
- distribution = replicate_alphas(clusters, weights, values, sample.distance)
675
+ distribution = replicate_alphas(clusters, weights, values, sample.distance)
676
+ if method == "percentile":
642
677
  intervals[name] = percentile_interval(
643
678
  name, points[name], distribution, confidence, alpha_min
644
679
  )
645
680
  else:
646
- statistic = _alpha_statistic(clusters, n_clusters, values, sample.distance)
681
+ theta_hat, jackknife = bca_parts(clusters, values, sample.distance)
647
682
  intervals[name] = bca_interval(
648
- name,
649
- points[name],
650
- statistic,
651
- n_clusters,
652
- n_resamples,
653
- confidence,
654
- seed,
655
- alpha_min,
683
+ name, points[name], distribution, theta_hat, jackknife, confidence, alpha_min
656
684
  )
657
685
  notes.append(UNITS_NOTE.format(level=level_used, within=" within strata" if strata else ""))
658
686
  if len(names) > 1:
@@ -795,7 +823,7 @@ def compare(
795
823
  weights = (
796
824
  draw_weights(np.random.default_rng(seed), plan_a, n_resamples)
797
825
  if method == "percentile"
798
- else None
826
+ else bca_weights(seed, plan_a.n_clusters, n_resamples)
799
827
  )
800
828
 
801
829
  arm_intervals: dict[str, dict[str, Interval]] = {"a": {}, "b": {}}
@@ -806,30 +834,22 @@ def compare(
806
834
  clusters_a = cluster_coincidences(sample_a, name, plan_a)
807
835
  clusters_b = cluster_coincidences(sample_b, name, plan_b)
808
836
  values_a, values_b = sample_a.inputs[name].values, sample_b.inputs[name].values
809
- if weights is not None:
810
- dist_a = replicate_alphas(clusters_a, weights, values_a, sample_a.distance)
811
- dist_b = replicate_alphas(clusters_b, weights, values_b, sample_b.distance)
837
+ dist_a = replicate_alphas(clusters_a, weights, values_a, sample_a.distance)
838
+ dist_b = replicate_alphas(clusters_b, weights, values_b, sample_b.distance)
839
+ if method == "percentile":
812
840
  arm_intervals["a"][name] = percentile_interval(name, point_a, dist_a, confidence)
813
841
  arm_intervals["b"][name] = percentile_interval(name, point_b, dist_b, confidence)
814
842
  differences[name] = percentile_interval(
815
843
  name, point_b - point_a, dist_b - dist_a, confidence
816
844
  )
817
845
  continue
818
- k = plan_a.n_clusters
819
- stat_a = _alpha_statistic(clusters_a, k, values_a, sample_a.distance)
820
- stat_b = _alpha_statistic(clusters_b, k, values_b, sample_b.distance)
821
-
822
- def stat_diff(
823
- idx: IntArray,
824
- stat_a: Callable[[IntArray], FloatArray] = stat_a,
825
- stat_b: Callable[[IntArray], FloatArray] = stat_b,
826
- ) -> FloatArray:
827
- return stat_b(idx) - stat_a(idx)
828
-
829
- args = (k, n_resamples, confidence, seed)
830
- arm_intervals["a"][name] = bca_interval(name, point_a, stat_a, *args)
831
- arm_intervals["b"][name] = bca_interval(name, point_b, stat_b, *args)
832
- differences[name] = bca_interval(name, point_b - point_a, stat_diff, *args)
846
+ theta_a, jack_a = bca_parts(clusters_a, values_a, sample_a.distance)
847
+ theta_b, jack_b = bca_parts(clusters_b, values_b, sample_b.distance)
848
+ arm_intervals["a"][name] = bca_interval(name, point_a, dist_a, theta_a, jack_a, confidence)
849
+ arm_intervals["b"][name] = bca_interval(name, point_b, dist_b, theta_b, jack_b, confidence)
850
+ differences[name] = bca_interval(
851
+ name, point_b - point_a, dist_b - dist_a, theta_b - theta_a, jack_b - jack_a, confidence
852
+ )
833
853
 
834
854
  recorded.extend(_interval_warnings(differences, n_resamples))
835
855
  within = " within strata" if strata else ""
@@ -116,8 +116,11 @@ class ReliabilityStudy:
116
116
  ratings:
117
117
  Validated long-form ratings.
118
118
  measurement:
119
- ``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``, or a
120
- :class:`~reliability_lab.alpha.Distance`.
119
+ ``"nominal"``, ``"ordinal"``, ``"interval"``, ``"ratio"``, ``"bipolar"``,
120
+ ``circular(U)``, or a :class:`~reliability_lab.alpha.Distance`. Bipolar
121
+ distances take the scale's two poles from the smallest and largest declared
122
+ values. Circular distances need the circumference U (24 for hours, 360 for
123
+ degrees), so the string ``"circular"`` is refused.
121
124
  values:
122
125
  The legal values. For ordinal data their order is the scale order. The
123
126
  domain is always declared: pass ``ratings.observed_values()`` to declare the
@@ -935,6 +938,11 @@ def _resolve_distance(measurement: str | Distance) -> Distance:
935
938
  return measurement
936
939
  if isinstance(measurement, str) and measurement in BUILTIN_DISTANCES:
937
940
  return BUILTIN_DISTANCES[measurement]
941
+ if measurement == "circular":
942
+ raise DesignError(
943
+ "circular data need the circle's circumference: pass measurement=circular(U), "
944
+ "for example circular(24) for hours or circular(360) for degrees"
945
+ )
938
946
  raise DesignError(
939
947
  f"measurement must be one of {list(BUILTIN_DISTANCES)} or a Distance; got {measurement!r}"
940
948
  )