reliability-lab 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/CHANGELOG.md +10 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/PKG-INFO +60 -4
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/README.md +58 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/design/roadmap.md +5 -3
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/simulated.py +4 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/splits.py +37 -34
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/pyproject.toml +23 -5
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/coders.py +2 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/coverage.py +2 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/paired.py +1 -1
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/report.py +10 -8
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/__init__.py +10 -0
- reliability_lab-0.3.0/src/reliability_lab/__main__.py +8 -0
- reliability_lab-0.3.0/src/reliability_lab/_version.py +1 -0
- reliability_lab-0.3.0/src/reliability_lab/cli.py +1023 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/data.py +90 -11
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/inference.py +7 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/llm.py +21 -9
- reliability_lab-0.3.0/src/reliability_lab/planning.py +1113 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/report.py +30 -5
- reliability_lab-0.3.0/src/reliability_lab/simulate.py +492 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/study.py +8 -4
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/helpers.py +11 -8
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/naive_bootstrap.py +8 -8
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/reference.py +9 -9
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_alpha.py +10 -5
- reliability_lab-0.3.0/tests/test_arrow_input.py +212 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_bca.py +5 -3
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_circular_bipolar.py +2 -2
- reliability_lab-0.3.0/tests/test_cli.py +680 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_compare.py +19 -2
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_data.py +38 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_inference_units.py +3 -3
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_kalpha.py +2 -1
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_krippendorff2011.py +1 -1
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_parity.py +4 -4
- reliability_lab-0.3.0/tests/test_planning.py +407 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_retries.py +5 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_simulate.py +80 -0
- reliability_lab-0.3.0/tests/test_smoke.py +20 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_split.py +1 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_trace.py +21 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/uv.lock +60 -2
- reliability_lab-0.2.0/src/reliability_lab/_version.py +0 -1
- reliability_lab-0.2.0/src/reliability_lab/simulate.py +0 -192
- reliability_lab-0.2.0/tests/test_smoke.py +0 -6
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.gitattributes +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.github/workflows/ci.yml +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.github/workflows/release.yml +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.gitignore +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.python-version +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/LICENSE +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/design/framing-v2.md +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/bootstrap-coverage.md +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_factor-dark.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_factor-light.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_n-dark.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_n-light.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_spread-dark.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_spread-light.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/power_by_n-dark.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/power_by_n-light.png +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/releasing.md +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/llm_judge.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/quickstart.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/__init__.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/grid.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coders/coverage.csv +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coders/run_info.json +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coverage.csv +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/paired/coverage.csv +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/paired/run_info.json +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/run_info.json +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/run_coverage.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/templates/bootstrap-coverage.md +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/alpha.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/design.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/exceptions.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/py.typed +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/split.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/__init__.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/krippendorff2011.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/llm_logs.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_coder_jackknife.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_coder_resampling.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_design.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_llm_end_to_end.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_noninferiority.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_properties.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_report.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_report_io.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_simulations.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_status_policies.py +0 -0
- {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_triage.py +0 -0
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
Planning, CI gates and Arrow input.
|
|
6
|
+
|
|
7
|
+
- `plan_krippendorff`, `plan_alpha`, `plan_non_inferiority`, and `SampleSizePlan`: analytical two-coder nominal sample-size rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989) plus CRN + monotone-probit simulation planning for minimum alpha thresholds and paired non-inferiority comparisons.
|
|
8
|
+
- `simulate_ratings`: optional latent-class parameters `hard_share`, `hard_penalty`, `cluster_size` (`group_id`), `cohesion`, `rho`, and informative `hard_failures` with population alpha calibration.
|
|
9
|
+
- `reliability-lab` CLI (`evaluate` and `compare`, plus `python -m reliability_lab`): TOML study specifications, CSV/TSV/Parquet log loading, `--policy` gating, threshold/verdict checks, JSON reports, and exit codes `0`–`3` (`--strict`).
|
|
10
|
+
- `RatingFrame.from_long` and `TraceSidecar.from_frame` accept any table that exports an Arrow stream (`__arrow_c_stream__`), such as polars DataFrames (1.3 and later), DuckDB relations (1.1 and later), pyarrow Tables and RecordBatchReaders. They are converted to pandas through pyarrow, with nulls as `None` or `NaN`. The same ratings give the same fingerprint as from pandas, except fractional `float32` values, timestamps and decimals.
|
|
11
|
+
- The `parquet` extra now needs pyarrow 16 or later, the first version that converts the `string_view` columns polars exports.
|
|
12
|
+
|
|
3
13
|
## 0.2.0
|
|
4
14
|
|
|
5
15
|
Decisions on a configuration change.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: reliability-lab
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
|
|
5
5
|
Project-URL: Repository, https://github.com/Sathapana/reliability-lab
|
|
6
6
|
Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
|
|
@@ -24,14 +24,14 @@ Requires-Dist: numpy>=1.26
|
|
|
24
24
|
Requires-Dist: pandas>=2.1
|
|
25
25
|
Requires-Dist: scipy>=1.15
|
|
26
26
|
Provides-Extra: parquet
|
|
27
|
-
Requires-Dist: pyarrow>=
|
|
27
|
+
Requires-Dist: pyarrow>=16; extra == 'parquet'
|
|
28
28
|
Provides-Extra: sklearn
|
|
29
29
|
Requires-Dist: scikit-learn>=1.4; extra == 'sklearn'
|
|
30
30
|
Description-Content-Type: text/markdown
|
|
31
31
|
|
|
32
32
|
# reliability-lab
|
|
33
33
|
|
|
34
|
-
**Status: alpha (0.
|
|
34
|
+
**Status: alpha (0.3.0).** The API may still change between releases.
|
|
35
35
|
|
|
36
36
|
reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
|
|
37
37
|
|
|
@@ -56,7 +56,7 @@ Mixed human-LLM panels are planned for 1.0 ([roadmap](https://github.com/Sathapa
|
|
|
56
56
|
pip install "reliability-lab[sklearn,parquet]"
|
|
57
57
|
```
|
|
58
58
|
|
|
59
|
-
The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet
|
|
59
|
+
The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet` and ratings from polars, DuckDB and pyarrow tables. For development, run `uv sync` and then `uv run pytest`.
|
|
60
60
|
|
|
61
61
|
## Example
|
|
62
62
|
|
|
@@ -93,6 +93,19 @@ per_unit.sort_values("d_o_contribution", ascending=False).head(10)
|
|
|
93
93
|
|
|
94
94
|
The `category_skew` diagnostic shows how evenly the ratings use the declared scale: the normalised entropy (1 when every value is used equally, 0 when one value holds them all) and the effective number of values. Alpha can be low while observed agreement is high when one value dominates.
|
|
95
95
|
|
|
96
|
+
### Polars, DuckDB and Arrow tables
|
|
97
|
+
|
|
98
|
+
`RatingFrame.from_long` and `TraceSidecar.from_frame` also accept any table that exports an Arrow stream (`__arrow_c_stream__`): a polars DataFrame, a DuckDB relation, a pyarrow Table or a RecordBatchReader. There is no need to call `to_pandas()` first.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
import duckdb
|
|
102
|
+
|
|
103
|
+
log = duckdb.sql("select * from 'ratings.parquet' where split = 'test'")
|
|
104
|
+
ratings = RatingFrame.from_long(log, unit="item", rater="annotator", value="label")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The table is converted through pyarrow (the `parquet` extra) and copied into pandas, so the conversion is convenient, not zero-copy. The same ratings give the same fingerprint whatever library they come from. Two exceptions: fractional `float32` values, and timestamps or decimals, can be written differently from their pandas forms.
|
|
108
|
+
|
|
96
109
|
## Simulated ratings
|
|
97
110
|
|
|
98
111
|
`simulate_ratings` makes synthetic coders for any measurement level, including `"bipolar"`, `circular(U)` and custom distances, at the alpha you choose. Use it to try the API, to see what a given alpha looks like in raw ratings, or to plan how many units a study needs.
|
|
@@ -245,6 +258,49 @@ The single-configuration designs (repeatability, prompt robustness, order robust
|
|
|
245
258
|
|
|
246
259
|
Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
|
|
247
260
|
|
|
261
|
+
## Sample-size planning
|
|
262
|
+
|
|
263
|
+
Plan sample sizes either from Krippendorff's analytical two-coder nominal rule or by simulation under the package's latent-class generator (with common random numbers and a monotone probit power curve):
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
from reliability_lab import plan_alpha, plan_krippendorff, plan_non_inferiority
|
|
267
|
+
|
|
268
|
+
# Analytical nominal rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989)
|
|
269
|
+
rule = plan_krippendorff(alpha_min=0.80, smallest_prevalence=0.25, significance=0.05)
|
|
270
|
+
|
|
271
|
+
# Simulation planning for arbitrary metrics, multi-coder dependence, clusters, or missingness
|
|
272
|
+
study_plan = plan_alpha(
|
|
273
|
+
"ordinal",
|
|
274
|
+
[1, 2, 3, 4, 5],
|
|
275
|
+
alpha=0.80,
|
|
276
|
+
alpha_min=0.667,
|
|
277
|
+
power=0.80,
|
|
278
|
+
n_coders=3,
|
|
279
|
+
random_state=42,
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
# Paired non-inferiority sample size (BCA at 96% confidence holds ~2.5% one-sided size)
|
|
283
|
+
ni_plan = plan_non_inferiority(
|
|
284
|
+
"nominal",
|
|
285
|
+
["a", "b", "c"],
|
|
286
|
+
alpha=0.80,
|
|
287
|
+
margin=0.05,
|
|
288
|
+
power=0.80,
|
|
289
|
+
random_state=42,
|
|
290
|
+
)
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
## CLI quality gates
|
|
294
|
+
|
|
295
|
+
Run CI reliability checks from the command line (`reliability-lab` or `python -m reliability_lab`) with a TOML study specification and CSV, TSV, or Parquet logs:
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
reliability-lab evaluate study.toml ratings.csv --min-alpha 0.70 --min-low 0.65 --strict --output report.json
|
|
299
|
+
reliability-lab compare study.toml ratings.parquet --a condition_id=baseline --b condition_id=candidate --margin 0.05 --test non_inferiority --output comparison.json
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Thresholds and comparison verdicts apply across every reported missing-data policy unless `--policy` selects one (`as_missing`, `as_category`, `sensitivity_pessimistic`, `sensitivity_optimistic`, or `sensitivity`). Exit codes are structured for CI: `0` pass, `1` usage/configuration/schema error, `2` threshold or verdict failure, and `3` passing thresholds with design or trace findings when `--strict` is set.
|
|
303
|
+
|
|
248
304
|
## Licence
|
|
249
305
|
|
|
250
306
|
BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# reliability-lab
|
|
2
2
|
|
|
3
|
-
**Status: alpha (0.
|
|
3
|
+
**Status: alpha (0.3.0).** The API may still change between releases.
|
|
4
4
|
|
|
5
5
|
reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
|
|
6
6
|
|
|
@@ -25,7 +25,7 @@ Mixed human-LLM panels are planned for 1.0 ([roadmap](https://github.com/Sathapa
|
|
|
25
25
|
pip install "reliability-lab[sklearn,parquet]"
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
-
The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet
|
|
28
|
+
The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet` and ratings from polars, DuckDB and pyarrow tables. For development, run `uv sync` and then `uv run pytest`.
|
|
29
29
|
|
|
30
30
|
## Example
|
|
31
31
|
|
|
@@ -62,6 +62,19 @@ per_unit.sort_values("d_o_contribution", ascending=False).head(10)
|
|
|
62
62
|
|
|
63
63
|
The `category_skew` diagnostic shows how evenly the ratings use the declared scale: the normalised entropy (1 when every value is used equally, 0 when one value holds them all) and the effective number of values. Alpha can be low while observed agreement is high when one value dominates.
|
|
64
64
|
|
|
65
|
+
### Polars, DuckDB and Arrow tables
|
|
66
|
+
|
|
67
|
+
`RatingFrame.from_long` and `TraceSidecar.from_frame` also accept any table that exports an Arrow stream (`__arrow_c_stream__`): a polars DataFrame, a DuckDB relation, a pyarrow Table or a RecordBatchReader. There is no need to call `to_pandas()` first.
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
import duckdb
|
|
71
|
+
|
|
72
|
+
log = duckdb.sql("select * from 'ratings.parquet' where split = 'test'")
|
|
73
|
+
ratings = RatingFrame.from_long(log, unit="item", rater="annotator", value="label")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The table is converted through pyarrow (the `parquet` extra) and copied into pandas, so the conversion is convenient, not zero-copy. The same ratings give the same fingerprint whatever library they come from. Two exceptions: fractional `float32` values, and timestamps or decimals, can be written differently from their pandas forms.
|
|
77
|
+
|
|
65
78
|
## Simulated ratings
|
|
66
79
|
|
|
67
80
|
`simulate_ratings` makes synthetic coders for any measurement level, including `"bipolar"`, `circular(U)` and custom distances, at the alpha you choose. Use it to try the API, to see what a given alpha looks like in raw ratings, or to plan how many units a study needs.
|
|
@@ -214,6 +227,49 @@ The single-configuration designs (repeatability, prompt robustness, order robust
|
|
|
214
227
|
|
|
215
228
|
Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
|
|
216
229
|
|
|
230
|
+
## Sample-size planning
|
|
231
|
+
|
|
232
|
+
Plan sample sizes either from Krippendorff's analytical two-coder nominal rule or by simulation under the package's latent-class generator (with common random numbers and a monotone probit power curve):
|
|
233
|
+
|
|
234
|
+
```python
|
|
235
|
+
from reliability_lab import plan_alpha, plan_krippendorff, plan_non_inferiority
|
|
236
|
+
|
|
237
|
+
# Analytical nominal rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989)
|
|
238
|
+
rule = plan_krippendorff(alpha_min=0.80, smallest_prevalence=0.25, significance=0.05)
|
|
239
|
+
|
|
240
|
+
# Simulation planning for arbitrary metrics, multi-coder dependence, clusters, or missingness
|
|
241
|
+
study_plan = plan_alpha(
|
|
242
|
+
"ordinal",
|
|
243
|
+
[1, 2, 3, 4, 5],
|
|
244
|
+
alpha=0.80,
|
|
245
|
+
alpha_min=0.667,
|
|
246
|
+
power=0.80,
|
|
247
|
+
n_coders=3,
|
|
248
|
+
random_state=42,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
# Paired non-inferiority sample size (BCA at 96% confidence holds ~2.5% one-sided size)
|
|
252
|
+
ni_plan = plan_non_inferiority(
|
|
253
|
+
"nominal",
|
|
254
|
+
["a", "b", "c"],
|
|
255
|
+
alpha=0.80,
|
|
256
|
+
margin=0.05,
|
|
257
|
+
power=0.80,
|
|
258
|
+
random_state=42,
|
|
259
|
+
)
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## CLI quality gates
|
|
263
|
+
|
|
264
|
+
Run CI reliability checks from the command line (`reliability-lab` or `python -m reliability_lab`) with a TOML study specification and CSV, TSV, or Parquet logs:
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
reliability-lab evaluate study.toml ratings.csv --min-alpha 0.70 --min-low 0.65 --strict --output report.json
|
|
268
|
+
reliability-lab compare study.toml ratings.parquet --a condition_id=baseline --b condition_id=candidate --margin 0.05 --test non_inferiority --output comparison.json
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Thresholds and comparison verdicts apply across every reported missing-data policy unless `--policy` selects one (`as_missing`, `as_category`, `sensitivity_pessimistic`, `sensitivity_optimistic`, or `sensitivity`). Exit codes are structured for CI: `0` pass, `1` usage/configuration/schema error, `2` threshold or verdict failure, and `3` passing thresholds with design or trace findings when `--strict` is set.
|
|
272
|
+
|
|
217
273
|
## Licence
|
|
218
274
|
|
|
219
275
|
BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.
|
|
@@ -14,7 +14,7 @@ Two decisions shape the order:
|
|
|
14
14
|
| Release | Theme | Contents | Gate |
|
|
15
15
|
| --- | --- | --- | --- |
|
|
16
16
|
| 0.2.0 (shipped) | Decisions on a configuration change | Non-inferiority and equivalence in `compare()` (2.2); BCa failure reasons and opt-in fallback (1.2); category-skew diagnostic (1.3); unit triage (1.4) | At `confidence=0.96`, rejection at the margin holds 2.5%; at 0.95 it is measured and reported (revised 30 Oct 2026, see the 0.2.0 gate); a test for each BCa failure cause |
|
|
17
|
-
| 0.3.0 | Planning, CI gates and Arrow input | Sample-size planning (2.3); CLI quality gates (4.2); Arrow input (4.1) | Planned sizes reach their stated power on fresh seeds; CLI runs the example fixtures end to end; polars, DuckDB and pyarrow inputs give the same ratings fingerprint as pandas |
|
|
17
|
+
| 0.3.0 (shipped) | Planning, CI gates and Arrow input | Sample-size planning (2.3); CLI quality gates (4.2); Arrow input (4.1) | Planned sizes reach their stated power on fresh seeds; CLI runs the example fixtures end to end; polars, DuckDB and pyarrow inputs give the same ratings fingerprint as pandas |
|
|
18
18
|
| 0.4.0 | Judge bias evidence | Within- and between-family alpha (3.2); unit covariates for length bias and self-enhancement (3.2); call telemetry, reduced (3.3) | Simulated same-family errors detected; fixtures for each diagnostic |
|
|
19
19
|
| 0.5.0 | Companion coefficients and plots | `metrics/` with Gwet's AC1 and AC2 (1.3); plotting extra (4.3) | Parity with irrCAC |
|
|
20
20
|
| 1.0.0 | Human-LLM alignment and a stable release | Mixed panels; alignment averaged over individual annotators and against a consensus label (3.1); stable API; frozen report schema; coverage simulations for interval, ratio and mixed-panel data | Mixed-panel end-to-end fixture with hand-checked values; paired intervals checked against a naive loop; reports from 1.0 read back by later releases; guidance regenerated from full-size runs |
|
|
@@ -114,9 +114,11 @@ A calibrated interval (a bootstrap-t, or a double bootstrap that sets the level
|
|
|
114
114
|
|
|
115
115
|
## 0.3.0: Planning, CI gates and Arrow input
|
|
116
116
|
|
|
117
|
+
Shipped in 0.3.0: every item below, with its gate passed.
|
|
118
|
+
|
|
117
119
|
**Sample-size planning (2.3).** A new `planning.py`; `design.py` holds coder definitions and design labels. The draft's inputs (target alpha, marginals, number of coders) do not determine how ratings are generated, so each plan names its generator:
|
|
118
120
|
|
|
119
|
-
1. For nominal data, the minimum-sample-size rule in Krippendorff's *Content Analysis
|
|
121
|
+
1. For nominal data, the minimum-sample-size rule in Krippendorff's *Content Analysis* (2nd ed. 2004, Table 11.1; 3rd ed. 2013, p. 323; 4th ed. 2019, Table 12.3) and Krippendorff (2011, Table 1, eq. via Bloch & Kraemer, 1989, eq. 3.7): the number of values needed for a given minimum alpha, smallest category probability and significance level.
|
|
120
122
|
2. For every metric, simulation with the latent-class "know or guess" generator of `simulations/`. That generator moves into the package, because the wheel ships `src/reliability_lab` only. The search over N uses common random numbers and a monotone fit, because a plain binary search on noisy power estimates can stop on noise.
|
|
121
123
|
3. The same machinery plans the non-inferiority test of 0.2.0: the number of units needed to show non-inferiority at margin δ when the true difference is zero.
|
|
122
124
|
|
|
@@ -252,7 +254,7 @@ At 95%, the paired intervals of `compare()` miss about 0.25 points too often per
|
|
|
252
254
|
|
|
253
255
|
## References
|
|
254
256
|
|
|
255
|
-
1. Krippendorff, K. *Content Analysis: An Introduction to Its Methodology.* Sage.
|
|
257
|
+
1. Krippendorff, K. *Content Analysis: An Introduction to Its Methodology.* Sage (2nd ed. 2004, Table 11.1; 3rd ed. 2013, p. 323; 4th ed. 2019, Table 12.3); Krippendorff, K. (2011). "Agreement and Information in the Reliability of Coding." *Communication Methods and Measures*, 5(2), 93–112, Table 1; Bloch, D. A., & Kraemer, H. C. (1989). "2 × 2 Kappa Coefficients: Measures of Agreement or Association." *Biometrics*, 45(1), 269–287, eq. 3.7.
|
|
256
258
|
2. Owen, A. B. (2007). [The pigeonhole bootstrap](https://doi.org/10.1214/07-AOAS122). Annals of Applied Statistics 1(2).
|
|
257
259
|
3. Calderon, N., Reichart, R., & Dror, R. (2025). [The Alternative Annotator Test for LLM-as-a-Judge](https://arxiv.org/pdf/2501.10970). ACL 2025.
|
|
258
260
|
4. Gwet, K. L. (2008). [Computing inter-rater reliability and its variance in the presence of high agreement](https://doi.org/10.1348/000711006X126600). British Journal of Mathematical and Statistical Psychology 61(1).
|
|
@@ -5,9 +5,11 @@ Run with ``uv run python examples/simulated.py``. Each level gets 300 units rate
|
|
|
5
5
|
misses. The alphas printed vary around 0.8 from draw to draw.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
from
|
|
8
|
+
from typing import Any
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
from reliability_lab import Distance, ReliabilityStudy, circular, simulate_ratings
|
|
11
|
+
|
|
12
|
+
LEVELS: list[tuple[str | Distance, list[Any]]] = [
|
|
11
13
|
("nominal", ["cat", "dog", "bird", "fish"]),
|
|
12
14
|
("ordinal", [1, 2, 3, 4, 5]),
|
|
13
15
|
("interval", [1, 2, 3, 4, 5, 6, 7]),
|
|
@@ -65,37 +65,40 @@ def study_of(df: pd.DataFrame) -> ReliabilityStudy:
|
|
|
65
65
|
)
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
test.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
print(
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
68
|
+
with tempfile.TemporaryDirectory(prefix="reliability-lab-") as tmpdir:
|
|
69
|
+
out = Path(tmpdir)
|
|
70
|
+
study = study_of(judge_log("prompt_v1", seed=1))
|
|
71
|
+
|
|
72
|
+
# 1. One fold of StratifiedGroupKFold(4) is an approximate 25% stratified group holdout.
|
|
73
|
+
dev, test = study.holdout(StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=7))
|
|
74
|
+
test.save_manifest(out / "split.json")
|
|
75
|
+
assert test.manifest is not None
|
|
76
|
+
print("realised split:", test.manifest.realised)
|
|
77
|
+
|
|
78
|
+
# ... tune the judge on `dev` only, then freeze it ...
|
|
79
|
+
|
|
80
|
+
# 2. Alpha on the test partition; the bootstrap resamples whole conversations.
|
|
81
|
+
result = test.alpha()
|
|
82
|
+
boot = result.bootstrap(scheme="units", n_resamples=2000, random_state=7)
|
|
83
|
+
print()
|
|
84
|
+
print(boot.summary())
|
|
85
|
+
|
|
86
|
+
# 3. Reports: strict JSON with provenance, and tidy Parquet tables.
|
|
87
|
+
result.to_json(out / "alpha_test.json")
|
|
88
|
+
boot.to_parquet(out / "bootstrap_test")
|
|
89
|
+
print("\nprovenance gaps:", *result.provenance.gaps if result.provenance else [], sep="\n - ")
|
|
90
|
+
|
|
91
|
+
# 4. The same split, by its saved ids, on a later log with a new prompt.
|
|
92
|
+
later = study_of(judge_log("prompt_v2", seed=2))
|
|
93
|
+
_, later_test = later.apply_manifest(out / "split.json")
|
|
94
|
+
print(f"\nprompt_v2 alpha on the same test units: {later_test.alpha().alpha:.3f}")
|
|
95
|
+
|
|
96
|
+
few_shot = sorted(set(dev.ratings.to_frame()["unit_id"]))[:3]
|
|
97
|
+
test.manifest.assert_no_leakage(few_shot)
|
|
98
|
+
try:
|
|
99
|
+
test.manifest.assert_no_leakage(
|
|
100
|
+
[*few_shot, sorted(set(test.ratings.to_frame()["unit_id"]))[0]]
|
|
101
|
+
)
|
|
102
|
+
except LeakageError as err:
|
|
103
|
+
print(f"\nleakage caught: {err}")
|
|
104
|
+
print(f"\nfiles written to {out}")
|
|
@@ -39,7 +39,10 @@ dependencies = [
|
|
|
39
39
|
|
|
40
40
|
[project.optional-dependencies]
|
|
41
41
|
sklearn = ["scikit-learn>=1.4"]
|
|
42
|
-
parquet = ["pyarrow>=
|
|
42
|
+
parquet = ["pyarrow>=16"]
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
reliability-lab = "reliability_lab.cli:main"
|
|
43
46
|
|
|
44
47
|
[project.urls]
|
|
45
48
|
Repository = "https://github.com/Sathapana/reliability-lab"
|
|
@@ -56,7 +59,7 @@ dev = [
|
|
|
56
59
|
"pandas-stubs>=2.1",
|
|
57
60
|
# Exercise the optional extras in tests; they stay optional at runtime.
|
|
58
61
|
"scikit-learn>=1.4",
|
|
59
|
-
"pyarrow>=
|
|
62
|
+
"pyarrow>=16",
|
|
60
63
|
]
|
|
61
64
|
# Reference implementations used only by the parity tests. `krippendorff` is
|
|
62
65
|
# GPL-licensed, so it must never become a runtime dependency. `irrCAC` is not
|
|
@@ -67,13 +70,19 @@ parity = [
|
|
|
67
70
|
"nltk>=3.8",
|
|
68
71
|
"evalica>=0.3",
|
|
69
72
|
]
|
|
73
|
+
# Arrow producers for the Arrow input tests: the first versions that export
|
|
74
|
+
# __arrow_c_stream__.
|
|
75
|
+
interop = [
|
|
76
|
+
"polars>=1.3",
|
|
77
|
+
"duckdb>=1.1",
|
|
78
|
+
]
|
|
70
79
|
# Figures for the coverage-simulation guidance (simulations/report.py).
|
|
71
80
|
sim = [
|
|
72
81
|
"matplotlib>=3.8",
|
|
73
82
|
]
|
|
74
83
|
|
|
75
84
|
[tool.uv]
|
|
76
|
-
default-groups = ["dev", "parity"]
|
|
85
|
+
default-groups = ["dev", "parity", "interop"]
|
|
77
86
|
|
|
78
87
|
[tool.hatch.version]
|
|
79
88
|
path = "src/reliability_lab/_version.py"
|
|
@@ -98,10 +107,19 @@ select = ["E", "F", "W", "I", "B", "UP", "SIM", "NPY", "RUF"]
|
|
|
98
107
|
[tool.mypy]
|
|
99
108
|
strict = true
|
|
100
109
|
python_version = "3.11"
|
|
101
|
-
files = ["src"]
|
|
110
|
+
files = ["src", "tests", "simulations", "examples"]
|
|
102
111
|
|
|
103
112
|
[[tool.mypy.overrides]]
|
|
104
|
-
module = [
|
|
113
|
+
module = [
|
|
114
|
+
"duckdb.*",
|
|
115
|
+
"evalica.*",
|
|
116
|
+
"irrCAC.*",
|
|
117
|
+
"krippendorff.*",
|
|
118
|
+
"matplotlib.*",
|
|
119
|
+
"nltk.*",
|
|
120
|
+
"scipy.*",
|
|
121
|
+
"sklearn.*",
|
|
122
|
+
]
|
|
105
123
|
ignore_missing_imports = true
|
|
106
124
|
|
|
107
125
|
[tool.pytest.ini_options]
|
|
@@ -127,8 +127,8 @@ def replicate(
|
|
|
127
127
|
boot_seed = int(np.random.default_rng([*seed, 7]).integers(0, 2**63 - 1))
|
|
128
128
|
intervals = {
|
|
129
129
|
scheme: result.bootstrap(
|
|
130
|
-
scheme=scheme,
|
|
131
|
-
level="unit" if scheme == "units" else None,
|
|
130
|
+
scheme=scheme,
|
|
131
|
+
level="unit" if scheme == "units" else None,
|
|
132
132
|
n_resamples=n_resamples,
|
|
133
133
|
random_state=boot_seed,
|
|
134
134
|
)["as_missing"]
|
|
@@ -154,10 +154,10 @@ def rate(
|
|
|
154
154
|
|
|
155
155
|
def guesses(size: tuple[int, ...]) -> np.ndarray:
|
|
156
156
|
if s.measurement == "nominal":
|
|
157
|
-
return rng.choice(n_values, size=size, p=np.asarray(s.prevalence))
|
|
157
|
+
return np.asarray(rng.choice(n_values, size=size, p=np.asarray(s.prevalence)))
|
|
158
158
|
shifts, p = ORDINAL_SHIFTS
|
|
159
159
|
base = np.broadcast_to(truth[:, None] if len(size) == 2 else truth, size)
|
|
160
|
-
return np.clip(base + rng.choice(shifts, size=size, p=p), 0, n_values - 1)
|
|
160
|
+
return np.asarray(np.clip(base + rng.choice(shifts, size=size, p=p), 0, n_values - 1))
|
|
161
161
|
|
|
162
162
|
knows = rng.random((n_units, m)) < accuracy
|
|
163
163
|
own = guesses((n_units, m))
|
|
@@ -205,7 +205,7 @@ class PairedCellResult:
|
|
|
205
205
|
"miss_above": float((defined["low"] > delta).mean()) if k else math.nan,
|
|
206
206
|
"mean_width": float((defined["high"] - defined["low"]).mean()),
|
|
207
207
|
"bias": float((group["point"].dropna() - delta).mean()),
|
|
208
|
-
"nominal_level": (1 - confidence) / 2,
|
|
208
|
+
"nominal_level": (1.0 - float(confidence)) / 2, # type: ignore[arg-type]
|
|
209
209
|
**self.truth,
|
|
210
210
|
}
|
|
211
211
|
if self.scenario.kind == "margin":
|
|
@@ -23,6 +23,8 @@ import matplotlib
|
|
|
23
23
|
matplotlib.use("Agg")
|
|
24
24
|
import matplotlib.pyplot as plt
|
|
25
25
|
import pandas as pd
|
|
26
|
+
from matplotlib.axes import Axes
|
|
27
|
+
from matplotlib.figure import Figure
|
|
26
28
|
|
|
27
29
|
ROOT = Path(__file__).resolve().parent.parent
|
|
28
30
|
RESULTS = ROOT / "simulations" / "results"
|
|
@@ -112,7 +114,7 @@ def _floor(coverage: pd.Series) -> float:
|
|
|
112
114
|
return min(0.6, float(int(coverage.min() * 20) / 20) - 0.05)
|
|
113
115
|
|
|
114
116
|
|
|
115
|
-
def _style(ax:
|
|
117
|
+
def _style(ax: Axes, t: dict[str, str]) -> None:
|
|
116
118
|
ax.set_facecolor(t["surface"])
|
|
117
119
|
ax.grid(True, axis="y", color=t["grid"], linewidth=1 * PX)
|
|
118
120
|
ax.set_axisbelow(True)
|
|
@@ -122,7 +124,7 @@ def _style(ax: plt.Axes, t: dict[str, str]) -> None:
|
|
|
122
124
|
ax.tick_params(colors=t["muted"], labelcolor=t["ink2"], length=0, labelsize=8.5)
|
|
123
125
|
|
|
124
126
|
|
|
125
|
-
def _legend(fig:
|
|
127
|
+
def _legend(fig: Figure, t: dict[str, str], handles: list[Any]) -> None:
|
|
126
128
|
legend = fig.legend(
|
|
127
129
|
handles=handles,
|
|
128
130
|
loc="upper center",
|
|
@@ -136,7 +138,7 @@ def _legend(fig: plt.Figure, t: dict[str, str], handles: list[Any]) -> None:
|
|
|
136
138
|
text.set_color(t["ink"])
|
|
137
139
|
|
|
138
140
|
|
|
139
|
-
def _series(ax:
|
|
141
|
+
def _series(ax: Axes, t: dict[str, str], x: list[int], y: list[float], scheme: str) -> Any:
|
|
140
142
|
(line,) = ax.plot(
|
|
141
143
|
x,
|
|
142
144
|
y,
|
|
@@ -424,12 +426,12 @@ TIMES = chr(0x00D7)
|
|
|
424
426
|
TAU = chr(0x03C4)
|
|
425
427
|
|
|
426
428
|
|
|
427
|
-
def _pct(x:
|
|
428
|
-
return MISSING if pd.isna(x) else f"{x:.{digits}%}"
|
|
429
|
+
def _pct(x: Any, digits: int = 1) -> str:
|
|
430
|
+
return MISSING if pd.isna(x) else f"{float(x):.{digits}%}"
|
|
429
431
|
|
|
430
432
|
|
|
431
|
-
def _num(x:
|
|
432
|
-
return MISSING if pd.isna(x) else f"{x:.{digits}f}"
|
|
433
|
+
def _num(x: Any, digits: int = 3) -> str:
|
|
434
|
+
return MISSING if pd.isna(x) else f"{float(x):.{digits}f}"
|
|
433
435
|
|
|
434
436
|
|
|
435
437
|
def _markdown(frame: pd.DataFrame) -> str:
|
|
@@ -550,7 +552,7 @@ def table_coders(data: pd.DataFrame) -> str:
|
|
|
550
552
|
|
|
551
553
|
|
|
552
554
|
def run_info_text(info: dict[str, Any], data: pd.DataFrame) -> str:
|
|
553
|
-
grid = info.get("grid")
|
|
555
|
+
grid = str(info.get("grid", ""))
|
|
554
556
|
truth = {"coders": "both_random", "paired": "true_delta"}.get(grid, "true_alpha")
|
|
555
557
|
mc_error = data.drop_duplicates("cell")[f"{truth}_mc_error"].max()
|
|
556
558
|
v = info["versions"]
|
|
@@ -18,6 +18,12 @@ from reliability_lab.exceptions import (
|
|
|
18
18
|
UndefinedAlphaError,
|
|
19
19
|
)
|
|
20
20
|
from reliability_lab.llm import TraceSidecar
|
|
21
|
+
from reliability_lab.planning import (
|
|
22
|
+
SampleSizePlan,
|
|
23
|
+
plan_alpha,
|
|
24
|
+
plan_krippendorff,
|
|
25
|
+
plan_non_inferiority,
|
|
26
|
+
)
|
|
21
27
|
from reliability_lab.report import (
|
|
22
28
|
AlphaEstimate,
|
|
23
29
|
AlphaResult,
|
|
@@ -62,6 +68,7 @@ __all__ = [
|
|
|
62
68
|
"ReliabilityLabError",
|
|
63
69
|
"ReliabilityStudy",
|
|
64
70
|
"ReliabilityWarning",
|
|
71
|
+
"SampleSizePlan",
|
|
65
72
|
"SchemaError",
|
|
66
73
|
"SplitManifest",
|
|
67
74
|
"Status",
|
|
@@ -70,5 +77,8 @@ __all__ = [
|
|
|
70
77
|
"UndefinedAlphaError",
|
|
71
78
|
"__version__",
|
|
72
79
|
"circular",
|
|
80
|
+
"plan_alpha",
|
|
81
|
+
"plan_krippendorff",
|
|
82
|
+
"plan_non_inferiority",
|
|
73
83
|
"simulate_ratings",
|
|
74
84
|
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|