reliability-lab 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/CHANGELOG.md +10 -0
  2. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/PKG-INFO +60 -4
  3. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/README.md +58 -2
  4. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/design/roadmap.md +5 -3
  5. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/simulated.py +4 -2
  6. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/splits.py +37 -34
  7. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/pyproject.toml +23 -5
  8. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/coders.py +2 -2
  9. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/coverage.py +2 -2
  10. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/paired.py +1 -1
  11. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/report.py +10 -8
  12. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/__init__.py +10 -0
  13. reliability_lab-0.3.0/src/reliability_lab/__main__.py +8 -0
  14. reliability_lab-0.3.0/src/reliability_lab/_version.py +1 -0
  15. reliability_lab-0.3.0/src/reliability_lab/cli.py +1023 -0
  16. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/data.py +90 -11
  17. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/inference.py +7 -2
  18. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/llm.py +21 -9
  19. reliability_lab-0.3.0/src/reliability_lab/planning.py +1113 -0
  20. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/report.py +30 -5
  21. reliability_lab-0.3.0/src/reliability_lab/simulate.py +492 -0
  22. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/study.py +8 -4
  23. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/helpers.py +11 -8
  24. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/naive_bootstrap.py +8 -8
  25. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/reference.py +9 -9
  26. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_alpha.py +10 -5
  27. reliability_lab-0.3.0/tests/test_arrow_input.py +212 -0
  28. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_bca.py +5 -3
  29. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_circular_bipolar.py +2 -2
  30. reliability_lab-0.3.0/tests/test_cli.py +680 -0
  31. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_compare.py +19 -2
  32. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_data.py +38 -0
  33. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_inference_units.py +3 -3
  34. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_kalpha.py +2 -1
  35. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_krippendorff2011.py +1 -1
  36. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_parity.py +4 -4
  37. reliability_lab-0.3.0/tests/test_planning.py +407 -0
  38. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_retries.py +5 -0
  39. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_simulate.py +80 -0
  40. reliability_lab-0.3.0/tests/test_smoke.py +20 -0
  41. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_split.py +1 -0
  42. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_trace.py +21 -0
  43. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/uv.lock +60 -2
  44. reliability_lab-0.2.0/src/reliability_lab/_version.py +0 -1
  45. reliability_lab-0.2.0/src/reliability_lab/simulate.py +0 -192
  46. reliability_lab-0.2.0/tests/test_smoke.py +0 -6
  47. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.gitattributes +0 -0
  48. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.github/workflows/ci.yml +0 -0
  49. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.github/workflows/release.yml +0 -0
  50. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.gitignore +0 -0
  51. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/.python-version +0 -0
  52. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/LICENSE +0 -0
  53. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/design/framing-v2.md +0 -0
  54. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/bootstrap-coverage.md +0 -0
  55. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_factor-dark.png +0 -0
  56. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_factor-light.png +0 -0
  57. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_n-dark.png +0 -0
  58. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_n-light.png +0 -0
  59. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_spread-dark.png +0 -0
  60. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/coverage_by_spread-light.png +0 -0
  61. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/power_by_n-dark.png +0 -0
  62. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/guidance/figures/power_by_n-light.png +0 -0
  63. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/docs/releasing.md +0 -0
  64. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/llm_judge.py +0 -0
  65. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/examples/quickstart.py +0 -0
  66. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/__init__.py +0 -0
  67. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/grid.py +0 -0
  68. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coders/coverage.csv +0 -0
  69. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coders/run_info.json +0 -0
  70. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/coverage.csv +0 -0
  71. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/paired/coverage.csv +0 -0
  72. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/paired/run_info.json +0 -0
  73. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/results/run_info.json +0 -0
  74. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/run_coverage.py +0 -0
  75. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/simulations/templates/bootstrap-coverage.md +0 -0
  76. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/alpha.py +0 -0
  77. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/design.py +0 -0
  78. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/exceptions.py +0 -0
  79. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/py.typed +0 -0
  80. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/src/reliability_lab/split.py +0 -0
  81. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/__init__.py +0 -0
  82. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/krippendorff2011.py +0 -0
  83. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/fixtures/llm_logs.py +0 -0
  84. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_coder_jackknife.py +0 -0
  85. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_coder_resampling.py +0 -0
  86. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_design.py +0 -0
  87. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_llm_end_to_end.py +0 -0
  88. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_noninferiority.py +0 -0
  89. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_properties.py +0 -0
  90. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_report.py +0 -0
  91. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_report_io.py +0 -0
  92. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_simulations.py +0 -0
  93. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_status_policies.py +0 -0
  94. {reliability_lab-0.2.0 → reliability_lab-0.3.0}/tests/test_triage.py +0 -0
@@ -1,5 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0
4
+
5
+ Planning, CI gates and Arrow input.
6
+
7
+ - `plan_krippendorff`, `plan_alpha`, `plan_non_inferiority`, and `SampleSizePlan`: analytical two-coder nominal sample-size rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989) plus CRN + monotone-probit simulation planning for minimum alpha thresholds and paired non-inferiority comparisons.
8
+ - `simulate_ratings`: optional latent-class parameters `hard_share`, `hard_penalty`, `cluster_size` (`group_id`), `cohesion`, `rho`, and informative `hard_failures` with population alpha calibration.
9
+ - `reliability-lab` CLI (`evaluate` and `compare`, plus `python -m reliability_lab`): TOML study specifications, CSV/TSV/Parquet log loading, `--policy` gating, threshold/verdict checks, JSON reports, and exit codes `0`–`3` (`--strict`).
10
+ - `RatingFrame.from_long` and `TraceSidecar.from_frame` accept any table that exports an Arrow stream (`__arrow_c_stream__`), such as polars DataFrames (1.3 and later), DuckDB relations (1.1 and later), pyarrow Tables and RecordBatchReaders. They are converted to pandas through pyarrow, with nulls as `None` or `NaN`. The same ratings give the same fingerprint as from pandas, except fractional `float32` values, timestamps and decimals.
11
+ - The `parquet` extra now needs pyarrow 16 or later, the first version that converts the `string_view` columns polars exports.
12
+
3
13
  ## 0.2.0
4
14
 
5
15
  Decisions on a configuration change.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: reliability-lab
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
5
5
  Project-URL: Repository, https://github.com/Sathapana/reliability-lab
6
6
  Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
@@ -24,14 +24,14 @@ Requires-Dist: numpy>=1.26
24
24
  Requires-Dist: pandas>=2.1
25
25
  Requires-Dist: scipy>=1.15
26
26
  Provides-Extra: parquet
27
- Requires-Dist: pyarrow>=15; extra == 'parquet'
27
+ Requires-Dist: pyarrow>=16; extra == 'parquet'
28
28
  Provides-Extra: sklearn
29
29
  Requires-Dist: scikit-learn>=1.4; extra == 'sklearn'
30
30
  Description-Content-Type: text/markdown
31
31
 
32
32
  # reliability-lab
33
33
 
34
- **Status: alpha (0.2.0).** The API may still change between releases.
34
+ **Status: alpha (0.3.0).** The API may still change between releases.
35
35
 
36
36
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
37
37
 
@@ -56,7 +56,7 @@ Mixed human-LLM panels are planned for 1.0 ([roadmap](https://github.com/Sathapa
56
56
  pip install "reliability-lab[sklearn,parquet]"
57
57
  ```
58
58
 
59
- The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet`. For development, run `uv sync` and then `uv run pytest`.
59
+ The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet` and ratings from polars, DuckDB and pyarrow tables. For development, run `uv sync` and then `uv run pytest`.
60
60
 
61
61
  ## Example
62
62
 
@@ -93,6 +93,19 @@ per_unit.sort_values("d_o_contribution", ascending=False).head(10)
93
93
 
94
94
  The `category_skew` diagnostic shows how evenly the ratings use the declared scale: the normalised entropy (1 when every value is used equally, 0 when one value holds them all) and the effective number of values. Alpha can be low while observed agreement is high when one value dominates.
95
95
 
96
+ ### Polars, DuckDB and Arrow tables
97
+
98
+ `RatingFrame.from_long` and `TraceSidecar.from_frame` also accept any table that exports an Arrow stream (`__arrow_c_stream__`): a polars DataFrame, a DuckDB relation, a pyarrow Table or a RecordBatchReader. There is no need to call `to_pandas()` first.
99
+
100
+ ```python
101
+ import duckdb
102
+
103
+ log = duckdb.sql("select * from 'ratings.parquet' where split = 'test'")
104
+ ratings = RatingFrame.from_long(log, unit="item", rater="annotator", value="label")
105
+ ```
106
+
107
+ The table is converted through pyarrow (the `parquet` extra) and copied into pandas, so the conversion is convenient, not zero-copy. The same ratings give the same fingerprint whatever library they come from. Two exceptions: fractional `float32` values, and timestamps or decimals, can be written differently from their pandas forms.
108
+
96
109
  ## Simulated ratings
97
110
 
98
111
  `simulate_ratings` makes synthetic coders for any measurement level, including `"bipolar"`, `circular(U)` and custom distances, at the alpha you choose. Use it to try the API, to see what a given alpha looks like in raw ratings, or to plan how many units a study needs.
@@ -245,6 +258,49 @@ The single-configuration designs (repeatability, prompt robustness, order robust
245
258
 
246
259
  Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
247
260
 
261
+ ## Sample-size planning
262
+
263
+ Plan sample sizes either from Krippendorff's analytical two-coder nominal rule or by simulation under the package's latent-class generator (with common random numbers and a monotone probit power curve):
264
+
265
+ ```python
266
+ from reliability_lab import plan_alpha, plan_krippendorff, plan_non_inferiority
267
+
268
+ # Analytical nominal rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989)
269
+ rule = plan_krippendorff(alpha_min=0.80, smallest_prevalence=0.25, significance=0.05)
270
+
271
+ # Simulation planning for arbitrary metrics, multi-coder dependence, clusters, or missingness
272
+ study_plan = plan_alpha(
273
+ "ordinal",
274
+ [1, 2, 3, 4, 5],
275
+ alpha=0.80,
276
+ alpha_min=0.667,
277
+ power=0.80,
278
+ n_coders=3,
279
+ random_state=42,
280
+ )
281
+
282
+ # Paired non-inferiority sample size (BCA at 96% confidence holds ~2.5% one-sided size)
283
+ ni_plan = plan_non_inferiority(
284
+ "nominal",
285
+ ["a", "b", "c"],
286
+ alpha=0.80,
287
+ margin=0.05,
288
+ power=0.80,
289
+ random_state=42,
290
+ )
291
+ ```
292
+
293
+ ## CLI quality gates
294
+
295
+ Run CI reliability checks from the command line (`reliability-lab` or `python -m reliability_lab`) with a TOML study specification and CSV, TSV, or Parquet logs:
296
+
297
+ ```bash
298
+ reliability-lab evaluate study.toml ratings.csv --min-alpha 0.70 --min-low 0.65 --strict --output report.json
299
+ reliability-lab compare study.toml ratings.parquet --a condition_id=baseline --b condition_id=candidate --margin 0.05 --test non_inferiority --output comparison.json
300
+ ```
301
+
302
+ Thresholds and comparison verdicts apply across every reported missing-data policy unless `--policy` selects one (`as_missing`, `as_category`, `sensitivity_pessimistic`, `sensitivity_optimistic`, or `sensitivity`). Exit codes are structured for CI: `0` pass, `1` usage/configuration/schema error, `2` threshold or verdict failure, and `3` passing thresholds with design or trace findings when `--strict` is set.
303
+
248
304
  ## Licence
249
305
 
250
306
  BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.
@@ -1,6 +1,6 @@
1
1
  # reliability-lab
2
2
 
3
- **Status: alpha (0.2.0).** The API may still change between releases.
3
+ **Status: alpha (0.3.0).** The API may still change between releases.
4
4
 
5
5
  reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
6
6
 
@@ -25,7 +25,7 @@ Mixed human-LLM panels are planned for 1.0 ([roadmap](https://github.com/Sathapa
25
25
  pip install "reliability-lab[sklearn,parquet]"
26
26
  ```
27
27
 
28
- The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet`. For development, run `uv sync` and then `uv run pytest`.
28
+ The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet` and ratings from polars, DuckDB and pyarrow tables. For development, run `uv sync` and then `uv run pytest`.
29
29
 
30
30
  ## Example
31
31
 
@@ -62,6 +62,19 @@ per_unit.sort_values("d_o_contribution", ascending=False).head(10)
62
62
 
63
63
  The `category_skew` diagnostic shows how evenly the ratings use the declared scale: the normalised entropy (1 when every value is used equally, 0 when one value holds them all) and the effective number of values. Alpha can be low while observed agreement is high when one value dominates.
64
64
 
65
+ ### Polars, DuckDB and Arrow tables
66
+
67
+ `RatingFrame.from_long` and `TraceSidecar.from_frame` also accept any table that exports an Arrow stream (`__arrow_c_stream__`): a polars DataFrame, a DuckDB relation, a pyarrow Table or a RecordBatchReader. There is no need to call `to_pandas()` first.
68
+
69
+ ```python
70
+ import duckdb
71
+
72
+ log = duckdb.sql("select * from 'ratings.parquet' where split = 'test'")
73
+ ratings = RatingFrame.from_long(log, unit="item", rater="annotator", value="label")
74
+ ```
75
+
76
+ The table is converted through pyarrow (the `parquet` extra) and copied into pandas, so the conversion is convenient, not zero-copy. The same ratings give the same fingerprint whatever library they come from. Two exceptions: fractional `float32` values, and timestamps or decimals, can be written differently from their pandas forms.
77
+
65
78
  ## Simulated ratings
66
79
 
67
80
  `simulate_ratings` makes synthetic coders for any measurement level, including `"bipolar"`, `circular(U)` and custom distances, at the alpha you choose. Use it to try the API, to see what a given alpha looks like in raw ratings, or to plan how many units a study needs.
@@ -214,6 +227,49 @@ The single-configuration designs (repeatability, prompt robustness, order robust
214
227
 
215
228
  Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](https://github.com/Sathapana/reliability-lab/blob/main/examples/llm_judge.py).
216
229
 
230
+ ## Sample-size planning
231
+
232
+ Plan sample sizes either from Krippendorff's analytical two-coder nominal rule or by simulation under the package's latent-class generator (with common random numbers and a monotone probit power curve):
233
+
234
+ ```python
235
+ from reliability_lab import plan_alpha, plan_krippendorff, plan_non_inferiority
236
+
237
+ # Analytical nominal rule (Krippendorff, 2011, Table 1; Bloch & Kraemer, 1989)
238
+ rule = plan_krippendorff(alpha_min=0.80, smallest_prevalence=0.25, significance=0.05)
239
+
240
+ # Simulation planning for arbitrary metrics, multi-coder dependence, clusters, or missingness
241
+ study_plan = plan_alpha(
242
+ "ordinal",
243
+ [1, 2, 3, 4, 5],
244
+ alpha=0.80,
245
+ alpha_min=0.667,
246
+ power=0.80,
247
+ n_coders=3,
248
+ random_state=42,
249
+ )
250
+
251
+ # Paired non-inferiority sample size (BCA at 96% confidence holds ~2.5% one-sided size)
252
+ ni_plan = plan_non_inferiority(
253
+ "nominal",
254
+ ["a", "b", "c"],
255
+ alpha=0.80,
256
+ margin=0.05,
257
+ power=0.80,
258
+ random_state=42,
259
+ )
260
+ ```
261
+
262
+ ## CLI quality gates
263
+
264
+ Run CI reliability checks from the command line (`reliability-lab` or `python -m reliability_lab`) with a TOML study specification and CSV, TSV, or Parquet logs:
265
+
266
+ ```bash
267
+ reliability-lab evaluate study.toml ratings.csv --min-alpha 0.70 --min-low 0.65 --strict --output report.json
268
+ reliability-lab compare study.toml ratings.parquet --a condition_id=baseline --b condition_id=candidate --margin 0.05 --test non_inferiority --output comparison.json
269
+ ```
270
+
271
+ Thresholds and comparison verdicts apply across every reported missing-data policy unless `--policy` selects one (`as_missing`, `as_category`, `sensitivity_pessimistic`, `sensitivity_optimistic`, or `sensitivity`). Exit codes are structured for CI: `0` pass, `1` usage/configuration/schema error, `2` threshold or verdict failure, and `3` passing thresholds with design or trace findings when `--strict` is set.
272
+
217
273
  ## Licence
218
274
 
219
275
  BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.
@@ -14,7 +14,7 @@ Two decisions shape the order:
14
14
  | Release | Theme | Contents | Gate |
15
15
  | --- | --- | --- | --- |
16
16
  | 0.2.0 (shipped) | Decisions on a configuration change | Non-inferiority and equivalence in `compare()` (2.2); BCa failure reasons and opt-in fallback (1.2); category-skew diagnostic (1.3); unit triage (1.4) | At `confidence=0.96`, rejection at the margin holds 2.5%; at 0.95 it is measured and reported (revised 30 Oct 2026, see the 0.2.0 gate); a test for each BCa failure cause |
17
- | 0.3.0 | Planning, CI gates and Arrow input | Sample-size planning (2.3); CLI quality gates (4.2); Arrow input (4.1) | Planned sizes reach their stated power on fresh seeds; CLI runs the example fixtures end to end; polars, DuckDB and pyarrow inputs give the same ratings fingerprint as pandas |
17
+ | 0.3.0 (shipped) | Planning, CI gates and Arrow input | Sample-size planning (2.3); CLI quality gates (4.2); Arrow input (4.1) | Planned sizes reach their stated power on fresh seeds; CLI runs the example fixtures end to end; polars, DuckDB and pyarrow inputs give the same ratings fingerprint as pandas |
18
18
  | 0.4.0 | Judge bias evidence | Within- and between-family alpha (3.2); unit covariates for length bias and self-enhancement (3.2); call telemetry, reduced (3.3) | Simulated same-family errors detected; fixtures for each diagnostic |
19
19
  | 0.5.0 | Companion coefficients and plots | `metrics/` with Gwet's AC1 and AC2 (1.3); plotting extra (4.3) | Parity with irrCAC |
20
20
  | 1.0.0 | Human-LLM alignment and a stable release | Mixed panels; alignment averaged over individual annotators and against a consensus label (3.1); stable API; frozen report schema; coverage simulations for interval, ratio and mixed-panel data | Mixed-panel end-to-end fixture with hand-checked values; paired intervals checked against a naive loop; reports from 1.0 read back by later releases; guidance regenerated from full-size runs |
@@ -114,9 +114,11 @@ A calibrated interval (a bootstrap-t, or a double bootstrap that sets the level
114
114
 
115
115
  ## 0.3.0: Planning, CI gates and Arrow input
116
116
 
117
+ Shipped in 0.3.0: every item below, with its gate passed.
118
+
117
119
  **Sample-size planning (2.3).** A new `planning.py`; `design.py` holds coder definitions and design labels. The draft's inputs (target alpha, marginals, number of coders) do not determine how ratings are generated, so each plan names its generator:
118
120
 
119
- 1. For nominal data, the minimum-sample-size rule in Krippendorff's *Content Analysis*: the number of values needed for a given minimum alpha, smallest category probability and significance level. The exact table and edition are to be verified before building.
121
+ 1. For nominal data, the minimum-sample-size rule in Krippendorff's *Content Analysis* (2nd ed. 2004, Table 11.1; 3rd ed. 2013, p. 323; 4th ed. 2019, Table 12.3) and Krippendorff (2011, Table 1, eq. via Bloch & Kraemer, 1989, eq. 3.7): the number of values needed for a given minimum alpha, smallest category probability and significance level.
120
122
  2. For every metric, simulation with the latent-class "know or guess" generator of `simulations/`. That generator moves into the package, because the wheel ships `src/reliability_lab` only. The search over N uses common random numbers and a monotone fit, because a plain binary search on noisy power estimates can stop on noise.
121
123
  3. The same machinery plans the non-inferiority test of 0.2.0: the number of units needed to show non-inferiority at margin δ when the true difference is zero.
122
124
 
@@ -252,7 +254,7 @@ At 95%, the paired intervals of `compare()` miss about 0.25 points too often per
252
254
 
253
255
  ## References
254
256
 
255
- 1. Krippendorff, K. *Content Analysis: An Introduction to Its Methodology.* Sage. Minimum-sample-size rule; edition and table to be verified.
257
+ 1. Krippendorff, K. *Content Analysis: An Introduction to Its Methodology.* Sage (2nd ed. 2004, Table 11.1; 3rd ed. 2013, p. 323; 4th ed. 2019, Table 12.3); Krippendorff, K. (2011). "Agreement and Information in the Reliability of Coding." *Communication Methods and Measures*, 5(2), 93–112, Table 1; Bloch, D. A., & Kraemer, H. C. (1989). "2 × 2 Kappa Coefficients: Measures of Agreement or Association." *Biometrics*, 45(1), 269–287, eq. 3.7.
256
258
  2. Owen, A. B. (2007). [The pigeonhole bootstrap](https://doi.org/10.1214/07-AOAS122). Annals of Applied Statistics 1(2).
257
259
  3. Calderon, N., Reichart, R., & Dror, R. (2025). [The Alternative Annotator Test for LLM-as-a-Judge](https://arxiv.org/pdf/2501.10970). ACL 2025.
258
260
  4. Gwet, K. L. (2008). [Computing inter-rater reliability and its variance in the presence of high agreement](https://doi.org/10.1348/000711006X126600). British Journal of Mathematical and Statistical Psychology 61(1).
@@ -5,9 +5,11 @@ Run with ``uv run python examples/simulated.py``. Each level gets 300 units rate
5
5
  misses. The alphas printed vary around 0.8 from draw to draw.
6
6
  """
7
7
 
8
- from reliability_lab import ReliabilityStudy, circular, simulate_ratings
8
+ from typing import Any
9
9
 
10
- LEVELS = [
10
+ from reliability_lab import Distance, ReliabilityStudy, circular, simulate_ratings
11
+
12
+ LEVELS: list[tuple[str | Distance, list[Any]]] = [
11
13
  ("nominal", ["cat", "dog", "bird", "fish"]),
12
14
  ("ordinal", [1, 2, 3, 4, 5]),
13
15
  ("interval", [1, 2, 3, 4, 5, 6, 7]),
@@ -65,37 +65,40 @@ def study_of(df: pd.DataFrame) -> ReliabilityStudy:
65
65
  )
66
66
 
67
67
 
68
- out = Path(tempfile.mkdtemp(prefix="reliability-lab-"))
69
- study = study_of(judge_log("prompt_v1", seed=1))
70
-
71
- # 1. One fold of StratifiedGroupKFold(4) is an approximate 25% stratified group holdout.
72
- dev, test = study.holdout(StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=7))
73
- test.save_manifest(out / "split.json")
74
- assert test.manifest is not None
75
- print("realised split:", test.manifest.realised)
76
-
77
- # ... tune the judge on `dev` only, then freeze it ...
78
-
79
- # 2. Alpha on the test partition; the bootstrap resamples whole conversations.
80
- result = test.alpha()
81
- boot = result.bootstrap(scheme="units", n_resamples=2000, random_state=7)
82
- print()
83
- print(boot.summary())
84
-
85
- # 3. Reports: strict JSON with provenance, and tidy Parquet tables.
86
- result.to_json(out / "alpha_test.json")
87
- boot.to_parquet(out / "bootstrap_test")
88
- print("\nprovenance gaps:", *result.provenance.gaps if result.provenance else [], sep="\n - ")
89
-
90
- # 4. The same split, by its saved ids, on a later log with a new prompt.
91
- later = study_of(judge_log("prompt_v2", seed=2))
92
- _, later_test = later.apply_manifest(out / "split.json")
93
- print(f"\nprompt_v2 alpha on the same test units: {later_test.alpha().alpha:.3f}")
94
-
95
- few_shot = sorted(set(dev.ratings.to_frame()["unit_id"]))[:3]
96
- test.manifest.assert_no_leakage(few_shot)
97
- try:
98
- test.manifest.assert_no_leakage([*few_shot, sorted(set(test.ratings.to_frame()["unit_id"]))[0]])
99
- except LeakageError as err:
100
- print(f"\nleakage caught: {err}")
101
- print(f"\nfiles written to {out}")
68
+ with tempfile.TemporaryDirectory(prefix="reliability-lab-") as tmpdir:
69
+ out = Path(tmpdir)
70
+ study = study_of(judge_log("prompt_v1", seed=1))
71
+
72
+ # 1. One fold of StratifiedGroupKFold(4) is an approximate 25% stratified group holdout.
73
+ dev, test = study.holdout(StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=7))
74
+ test.save_manifest(out / "split.json")
75
+ assert test.manifest is not None
76
+ print("realised split:", test.manifest.realised)
77
+
78
+ # ... tune the judge on `dev` only, then freeze it ...
79
+
80
+ # 2. Alpha on the test partition; the bootstrap resamples whole conversations.
81
+ result = test.alpha()
82
+ boot = result.bootstrap(scheme="units", n_resamples=2000, random_state=7)
83
+ print()
84
+ print(boot.summary())
85
+
86
+ # 3. Reports: strict JSON with provenance, and tidy Parquet tables.
87
+ result.to_json(out / "alpha_test.json")
88
+ boot.to_parquet(out / "bootstrap_test")
89
+ print("\nprovenance gaps:", *result.provenance.gaps if result.provenance else [], sep="\n - ")
90
+
91
+ # 4. The same split, by its saved ids, on a later log with a new prompt.
92
+ later = study_of(judge_log("prompt_v2", seed=2))
93
+ _, later_test = later.apply_manifest(out / "split.json")
94
+ print(f"\nprompt_v2 alpha on the same test units: {later_test.alpha().alpha:.3f}")
95
+
96
+ few_shot = sorted(set(dev.ratings.to_frame()["unit_id"]))[:3]
97
+ test.manifest.assert_no_leakage(few_shot)
98
+ try:
99
+ test.manifest.assert_no_leakage(
100
+ [*few_shot, sorted(set(test.ratings.to_frame()["unit_id"]))[0]]
101
+ )
102
+ except LeakageError as err:
103
+ print(f"\nleakage caught: {err}")
104
+ print(f"\nfiles written to {out}")
@@ -39,7 +39,10 @@ dependencies = [
39
39
 
40
40
  [project.optional-dependencies]
41
41
  sklearn = ["scikit-learn>=1.4"]
42
- parquet = ["pyarrow>=15"]
42
+ parquet = ["pyarrow>=16"]
43
+
44
+ [project.scripts]
45
+ reliability-lab = "reliability_lab.cli:main"
43
46
 
44
47
  [project.urls]
45
48
  Repository = "https://github.com/Sathapana/reliability-lab"
@@ -56,7 +59,7 @@ dev = [
56
59
  "pandas-stubs>=2.1",
57
60
  # Exercise the optional extras in tests; they stay optional at runtime.
58
61
  "scikit-learn>=1.4",
59
- "pyarrow>=15",
62
+ "pyarrow>=16",
60
63
  ]
61
64
  # Reference implementations used only by the parity tests. `krippendorff` is
62
65
  # GPL-licensed, so it must never become a runtime dependency. `irrCAC` is not
@@ -67,13 +70,19 @@ parity = [
67
70
  "nltk>=3.8",
68
71
  "evalica>=0.3",
69
72
  ]
73
+ # Arrow producers for the Arrow input tests: the first versions that export
74
+ # __arrow_c_stream__.
75
+ interop = [
76
+ "polars>=1.3",
77
+ "duckdb>=1.1",
78
+ ]
70
79
  # Figures for the coverage-simulation guidance (simulations/report.py).
71
80
  sim = [
72
81
  "matplotlib>=3.8",
73
82
  ]
74
83
 
75
84
  [tool.uv]
76
- default-groups = ["dev", "parity"]
85
+ default-groups = ["dev", "parity", "interop"]
77
86
 
78
87
  [tool.hatch.version]
79
88
  path = "src/reliability_lab/_version.py"
@@ -98,10 +107,19 @@ select = ["E", "F", "W", "I", "B", "UP", "SIM", "NPY", "RUF"]
98
107
  [tool.mypy]
99
108
  strict = true
100
109
  python_version = "3.11"
101
- files = ["src"]
110
+ files = ["src", "tests", "simulations", "examples"]
102
111
 
103
112
  [[tool.mypy.overrides]]
104
- module = ["scipy.*"]
113
+ module = [
114
+ "duckdb.*",
115
+ "evalica.*",
116
+ "irrCAC.*",
117
+ "krippendorff.*",
118
+ "matplotlib.*",
119
+ "nltk.*",
120
+ "scipy.*",
121
+ "sklearn.*",
122
+ ]
105
123
  ignore_missing_imports = true
106
124
 
107
125
  [tool.pytest.ini_options]
@@ -127,8 +127,8 @@ def replicate(
127
127
  boot_seed = int(np.random.default_rng([*seed, 7]).integers(0, 2**63 - 1))
128
128
  intervals = {
129
129
  scheme: result.bootstrap(
130
- scheme=scheme, # type: ignore[arg-type]
131
- level="unit" if scheme == "units" else None, # type: ignore[arg-type]
130
+ scheme=scheme,
131
+ level="unit" if scheme == "units" else None,
132
132
  n_resamples=n_resamples,
133
133
  random_state=boot_seed,
134
134
  )["as_missing"]
@@ -154,10 +154,10 @@ def rate(
154
154
 
155
155
  def guesses(size: tuple[int, ...]) -> np.ndarray:
156
156
  if s.measurement == "nominal":
157
- return rng.choice(n_values, size=size, p=np.asarray(s.prevalence))
157
+ return np.asarray(rng.choice(n_values, size=size, p=np.asarray(s.prevalence)))
158
158
  shifts, p = ORDINAL_SHIFTS
159
159
  base = np.broadcast_to(truth[:, None] if len(size) == 2 else truth, size)
160
- return np.clip(base + rng.choice(shifts, size=size, p=p), 0, n_values - 1)
160
+ return np.asarray(np.clip(base + rng.choice(shifts, size=size, p=p), 0, n_values - 1))
161
161
 
162
162
  knows = rng.random((n_units, m)) < accuracy
163
163
  own = guesses((n_units, m))
@@ -205,7 +205,7 @@ class PairedCellResult:
205
205
  "miss_above": float((defined["low"] > delta).mean()) if k else math.nan,
206
206
  "mean_width": float((defined["high"] - defined["low"]).mean()),
207
207
  "bias": float((group["point"].dropna() - delta).mean()),
208
- "nominal_level": (1 - confidence) / 2,
208
+ "nominal_level": (1.0 - float(confidence)) / 2, # type: ignore[arg-type]
209
209
  **self.truth,
210
210
  }
211
211
  if self.scenario.kind == "margin":
@@ -23,6 +23,8 @@ import matplotlib
23
23
  matplotlib.use("Agg")
24
24
  import matplotlib.pyplot as plt
25
25
  import pandas as pd
26
+ from matplotlib.axes import Axes
27
+ from matplotlib.figure import Figure
26
28
 
27
29
  ROOT = Path(__file__).resolve().parent.parent
28
30
  RESULTS = ROOT / "simulations" / "results"
@@ -112,7 +114,7 @@ def _floor(coverage: pd.Series) -> float:
112
114
  return min(0.6, float(int(coverage.min() * 20) / 20) - 0.05)
113
115
 
114
116
 
115
- def _style(ax: plt.Axes, t: dict[str, str]) -> None:
117
+ def _style(ax: Axes, t: dict[str, str]) -> None:
116
118
  ax.set_facecolor(t["surface"])
117
119
  ax.grid(True, axis="y", color=t["grid"], linewidth=1 * PX)
118
120
  ax.set_axisbelow(True)
@@ -122,7 +124,7 @@ def _style(ax: plt.Axes, t: dict[str, str]) -> None:
122
124
  ax.tick_params(colors=t["muted"], labelcolor=t["ink2"], length=0, labelsize=8.5)
123
125
 
124
126
 
125
- def _legend(fig: plt.Figure, t: dict[str, str], handles: list[Any]) -> None:
127
+ def _legend(fig: Figure, t: dict[str, str], handles: list[Any]) -> None:
126
128
  legend = fig.legend(
127
129
  handles=handles,
128
130
  loc="upper center",
@@ -136,7 +138,7 @@ def _legend(fig: plt.Figure, t: dict[str, str], handles: list[Any]) -> None:
136
138
  text.set_color(t["ink"])
137
139
 
138
140
 
139
- def _series(ax: plt.Axes, t: dict[str, str], x: list[int], y: list[float], scheme: str) -> Any:
141
+ def _series(ax: Axes, t: dict[str, str], x: list[int], y: list[float], scheme: str) -> Any:
140
142
  (line,) = ax.plot(
141
143
  x,
142
144
  y,
@@ -424,12 +426,12 @@ TIMES = chr(0x00D7)
424
426
  TAU = chr(0x03C4)
425
427
 
426
428
 
427
- def _pct(x: float, digits: int = 1) -> str:
428
- return MISSING if pd.isna(x) else f"{x:.{digits}%}"
429
+ def _pct(x: Any, digits: int = 1) -> str:
430
+ return MISSING if pd.isna(x) else f"{float(x):.{digits}%}"
429
431
 
430
432
 
431
- def _num(x: float, digits: int = 3) -> str:
432
- return MISSING if pd.isna(x) else f"{x:.{digits}f}"
433
+ def _num(x: Any, digits: int = 3) -> str:
434
+ return MISSING if pd.isna(x) else f"{float(x):.{digits}f}"
433
435
 
434
436
 
435
437
  def _markdown(frame: pd.DataFrame) -> str:
@@ -550,7 +552,7 @@ def table_coders(data: pd.DataFrame) -> str:
550
552
 
551
553
 
552
554
  def run_info_text(info: dict[str, Any], data: pd.DataFrame) -> str:
553
- grid = info.get("grid")
555
+ grid = str(info.get("grid", ""))
554
556
  truth = {"coders": "both_random", "paired": "true_delta"}.get(grid, "true_alpha")
555
557
  mc_error = data.drop_duplicates("cell")[f"{truth}_mc_error"].max()
556
558
  v = info["versions"]
@@ -18,6 +18,12 @@ from reliability_lab.exceptions import (
18
18
  UndefinedAlphaError,
19
19
  )
20
20
  from reliability_lab.llm import TraceSidecar
21
+ from reliability_lab.planning import (
22
+ SampleSizePlan,
23
+ plan_alpha,
24
+ plan_krippendorff,
25
+ plan_non_inferiority,
26
+ )
21
27
  from reliability_lab.report import (
22
28
  AlphaEstimate,
23
29
  AlphaResult,
@@ -62,6 +68,7 @@ __all__ = [
62
68
  "ReliabilityLabError",
63
69
  "ReliabilityStudy",
64
70
  "ReliabilityWarning",
71
+ "SampleSizePlan",
65
72
  "SchemaError",
66
73
  "SplitManifest",
67
74
  "Status",
@@ -70,5 +77,8 @@ __all__ = [
70
77
  "UndefinedAlphaError",
71
78
  "__version__",
72
79
  "circular",
80
+ "plan_alpha",
81
+ "plan_krippendorff",
82
+ "plan_non_inferiority",
73
83
  "simulate_ratings",
74
84
  ]
@@ -0,0 +1,8 @@
1
+ """Module entry point for ``python -m reliability_lab``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from reliability_lab.cli import main
6
+
7
+ if __name__ == "__main__":
8
+ raise SystemExit(main())
@@ -0,0 +1 @@
1
+ __version__ = "0.3.0"