reliability-lab 0.1.0__tar.gz

reliability_lab-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

reliability_lab-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+
+jobs:
+  test:
+    name: ${{ matrix.os }} / py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: uv sync --all-groups --locked
+      - name: Lint
+        run: uv run ruff check
+      - name: Format
+        run: uv run ruff format --check
+      - name: Type-check
+        run: uv run mypy
+      - name: Test
+        run: uv run pytest
+
+  # irrCAC pins scipy==1.12.0, below this package's floor (1.15, for the bootstrap's
+  # rng keyword). Its parity test only needs the point estimates, so it runs in a
+  # throwaway environment with the package installed without its dependencies.
+  parity-irrcac:
+    name: irrCAC parity (isolated)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v6
+      - name: Build the isolated environment
+        run: |
+          uv venv .venv-irrcac --python 3.12
+          uv pip install --python .venv-irrcac irrcac pytest hypothesis
+          uv pip install --python .venv-irrcac --no-deps .
+      - name: Run irrCAC parity
+        run: >-
+          .venv-irrcac/bin/python -m pytest tests/test_parity.py -m parity -k irrcac
+          -p no:cacheprovider

reliability_lab-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,65 @@
+name: Release
+
+# Publishes on a version tag (v0.1.0a1, v0.1.0, ...) with PyPI trusted publishing:
+# no API tokens are stored. See docs/releasing.md for the one-time setup.
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    name: Build and check the distributions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v6
+      - name: Check the tag matches the package version
+        run: |
+          version=$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' src/reliability_lab/_version.py)
+          if [ "v$version" != "$GITHUB_REF_NAME" ]; then
+            echo "tag $GITHUB_REF_NAME does not match package version $version" >&2
+            exit 1
+          fi
+      - name: Build
+        run: uv build
+      - name: Check metadata
+        run: uvx twine check --strict dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/reliability-lab
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: publish-testpypi
+    runs-on: ubuntu-latest
+    # Give this environment a required reviewer to approve each release by hand.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/reliability-lab
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

reliability_lab-0.1.0/.gitignore

@@ -0,0 +1,28 @@
+# Byte-compiled / cache
+__pycache__/
+*.py[cod]
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.hypothesis/
+
+# Environments
+.venv/
+venv/
+
+# Build artefacts
+build/
+dist/
+*.egg-info/
+
+# Coverage
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
+# Editors / OS
+.vscode/
+.idea/
+.DS_Store
+Thumbs.db

reliability_lab-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

reliability_lab-0.1.0/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+
+## Unreleased
+
+## 0.1.0
+
+LLM judge workflows.
+
+- `TraceSidecar`: one row per judge configuration (`condition_id`). It records provider, prompt hash, rubric and parser versions, temperature, seed, determinism, candidate order, request time and retry policy.
+- Each study checks its trace against its design. Configurations that differ in more than the design compares raise a `TraceWarning`; missing conditions or fields become provenance gaps. Comparisons report the model and trace fields that changed between the arms (`configuration_diff`).
+- The single-configuration designs (repeatability, prompt robustness, order robustness) can hold several judge configurations for `compare()`. Each configuration is checked on its own, and `alpha()` refuses to pool them.
+- `order_robustness` design for pairwise judges whose candidates are shown in swapped orders.
+- Diagnostics on every result (`AlphaResult.diagnostics`):
+  - `coder_agreement` (pairwise exact agreement);
+  - `run_agreement` (repeatability);
+  - `order_swap` (reversal rate, and position bias with a Wilson interval).
+- Retries: a new `attempt` field in `RatingFrame.from_long`. Retried calls must be declared with `retries="last"` or `retries="first_observed"`. Attempts are grouped by call (unit, rater, model, condition, run), so repeated runs are never treated as retries. Coverage reports attempt-level and final error rates.
+- Provenance records the trace rows of the conditions in use and the retries declaration.
+
+## 0.1.0a1
+
+First public release.
+
+- Project scaffold: packaging, BSD-3-Clause licence, CI on Linux, Windows and macOS.
+- Core engine: `RatingFrame` long-form schema with rating status, `CoderSpec` and design labels, Krippendorff's alpha (nominal, ordinal, interval, ratio, custom distances) with white-box diagnostics, and missing-data policies (`as_missing`, `as_category`, `sensitivity`).
+- Bootstrap inference: `AlphaResult.bootstrap(scheme=...)` with a required scheme.
+  - `units` resamples units, or groups, optionally within strata, and recomputes D_e per replicate.
+  - `kalpha` follows Krippendorff's algorithm, resampling pairs of values with D_e fixed.
+  - Percentile intervals are the default; BCa is opt-in and flagged experimental.
+  - Failed replicates are counted and reported, the seed is always recorded, and an optional `alpha_min` reports how many replicates fall below it.
+- Paired comparisons: `ReliabilityStudy.compare(a=..., b=...)` places the interval on Δα = α_b − α_a, using the same resampled units for both configurations.
+- Group-safe splits:
+  - `ReliabilityStudy.holdout()` and `cv()` take any scikit-learn style splitter. The splitter sees one sample per group, so groups and repeated LLM calls never cross partitions.
+  - When a group's units span several strata, `group_stratum` must be given.
+  - `SplitManifest` saves partition ids, the splitter, realised sizes and fingerprints; `apply_manifest()` reapplies them to a later log.
+  - `assert_no_leakage()` checks that few-shot examples come from the dev partition.
+- Reports:
+  - versioned JSON (`to_json`) with provenance for every result type: software versions, ratings fingerprint, declarations, split and known gaps;
+  - tidy tables (`tables`) and Parquet output (`to_parquet`, with the `parquet` extra).
+- Tag-triggered release workflow using PyPI trusted publishing.
+- New exceptions and warnings: `BootstrapWarning`, `LeakageError`. The scipy floor is now 1.15.

reliability_lab-0.1.0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Sathapana Chawananon
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

reliability_lab-0.1.0/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.5
+Name: reliability-lab
+Version: 0.1.0
+Summary: Study-design library for the reliability of LLM and human judgments, built on Krippendorff's alpha.
+Project-URL: Repository, https://github.com/Sathapana/reliability-lab
+Project-URL: Issues, https://github.com/Sathapana/reliability-lab/issues
+Author: Sathapana Chawananon
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Keywords: alpha,annotation,evaluation,inter-rater reliability,krippendorff,llm-as-a-judge
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.1
+Requires-Dist: scipy>=1.15
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=15; extra == 'parquet'
+Provides-Extra: sklearn
+Requires-Dist: scikit-learn>=1.4; extra == 'sklearn'
+Description-Content-Type: text/markdown
+
+# reliability-lab
+
+**Status: alpha (0.1.0).** The API may still change between releases.
+
+reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
+
+- a validated long-form ratings schema in which every rating has an explicit status (`observed`, `abstained`, `inapplicable`, `parse_error`, `provider_error`);
+- declared measurement semantics: level, value domain, coder definition and design label;
+- white-box output: observed and expected disagreement, coincidence matrices and per-unit contributions;
+- explicit missing-data policies, because LLM failures are rarely unrelated to the item;
+- bootstrap intervals by a scheme you name: whole units or groups resampled (`scheme="units"`), or Krippendorff's KALPHA algorithm (`scheme="kalpha"`);
+- paired comparisons of two configurations on the same units, with the interval on the difference;
+- group-safe train/test splits that work with any scikit-learn splitter, with a saved manifest that reproduces the split;
+- versioned JSON and Parquet reports with provenance;
+- LLM judge workflows: a trace sidecar of judge configurations, declared retry handling, and evidence specific to each design (run agreement, order swaps and position bias).
+
+Mixed human-LLM panels and coverage simulations for the bootstrap schemes are next on the roadmap ([design document](docs/design/framing-v2.md)).
+
+> **Reliability is not validity.** Coders can agree consistently and still be wrong in the same way, for example because of a flawed rubric or a bias they share. When results come from different designs (repeatability, inter-model agreement, human panels), they are reported as different quantities and are never pooled.
+
+## Install
+
+```bash
+pip install "reliability-lab[sklearn,parquet]"
+```
+
+The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet`. For development, run `uv sync` and then `uv run pytest`.
+
+## Example
+
+```python
+import pandas as pd
+from reliability_lab import CoderSpec, RatingFrame, ReliabilityStudy
+
+df = pd.DataFrame(
+    {
+        "item": ["u1", "u1", "u2", "u2", "u3", "u3"],
+        "annotator": ["ann", "bob", "ann", "bob", "ann", "bob"],
+        "label": ["pos", "pos", "neg", "pos", "neg", "neg"],
+    }
+)
+
+ratings = RatingFrame.from_long(df, unit="item", rater="annotator", value="label")
+study = ReliabilityStudy(
+    ratings,
+    measurement="nominal",
+    values=["neg", "pos"],
+    coders=CoderSpec(by="rater_id"),
+    design="human_panel",
+)
+result = study.alpha()
+print(result.summary())
+```
+
+## Intervals and comparisons
+
+Every interval names its scheme, because the two schemes answer different questions. `scheme` is a required argument.
+
+```python
+boot = result.bootstrap(scheme="units", n_resamples=2000, random_state=7)
+print(boot.summary())  # 95% percentile intervals for every missing-data policy
+```
+
+- `scheme="units"` resamples whole units, and whole groups when a `group` column is declared. It can also resample within declared strata (`strata=True`), and it recomputes the expected disagreement in every replicate. `method="bca"` is available as an experimental option.
+- `scheme="kalpha"` reproduces Krippendorff's KALPHA algorithm. It resamples pairs of values and holds the expected disagreement fixed.
+
+To compare two configurations of a judge, for example two prompts, select each one's rows. Both must cover the same units. Every replicate resamples the same units for both, and the interval is on the difference:
+
+```python
+diff = study.compare(
+    a={"condition_id": "prompt_v1"},
+    b={"condition_id": "prompt_v2"},
+    n_resamples=2000,
+    random_state=7,
+)
+print(diff.summary())  # alpha_b - alpha_a with a paired interval
+```
+
+## Splits and reports
+
+Teams tune the judge (prompt, few-shot examples, rubric) and should report reliability on units the tuning never saw. So split before tuning, freeze the judge, then estimate on the test partition.
+
+When groups are declared, the splitter sees one sample per group (a conversation, say), so groups and repeated LLM calls never cross partitions:
+
+```python
+from sklearn.model_selection import StratifiedGroupKFold
+
+dev, test = study.holdout(
+    StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=7),
+    group_stratum="majority",  # needed only when a group's units span several strata
+)
+test.save_manifest("split.json")  # the ids of every partition, not just the seed
+
+# ... tune and freeze the judge on dev ...
+
+result = test.alpha()
+result.to_json("alpha_test.json")  # strict JSON with provenance
+result.to_parquet("alpha_test/")  # tidy tables
+
+dev2, test2 = later_study.apply_manifest("split.json")  # same units, new judge runs
+test.manifest.assert_no_leakage(few_shot_units)  # raises LeakageError
+```
+
+See [`examples/splits.py`](examples/splits.py) for a full run. Releases are described in [docs/releasing.md](docs/releasing.md).
+
+## LLM judge workflows
+
+A `condition_id` names a frozen judge configuration. A trace sidecar records what each configuration was: provider, prompt hash, rubric and parser versions, temperature, seed, and candidate order for pairwise judges. With it, a study checks that its coders differ only in what the design compares. For example, if prompt-robustness conditions also differ in temperature, it raises a `TraceWarning`. Whatever the trace leaves out is listed among the provenance gaps.
+
+```python
+from reliability_lab import TraceSidecar
+
+trace = TraceSidecar.from_frame(
+    configs,
+    condition="condition_id",
+    prompt_hash="prompt_sha",
+    temperature="temperature",
+    candidate_order="order",  # "A,B" / "B,A"
+)
+study = ReliabilityStudy(
+    ratings,  # RatingFrame.from_long(..., attempt="attempt")
+    measurement="nominal",
+    values=["A", "tie", "B"],  # labels name the winning candidate, not its position
+    coders="condition_id",
+    design="order_robustness",
+    trace=trace,
+    retries="last",  # required when calls were retried
+)
+result = study.alpha()
+result.diagnostics["order_swap"].values  # reversal rate, P(first shown wins) with a Wilson interval
+result.trace_findings  # confounds and caveats from the trace
+```
+
+| Design | Coders are | Evidence beside alpha |
+| --- | --- | --- |
+| `repeatability` | repeated runs of one configuration (`run_id`) | run agreement: units where every run agrees |
+| `prompt_robustness` | prompt conditions of one model | pairwise agreement between conditions |
+| `inter_model` | different models | pairwise agreement; family and configuration checks |
+| `order_robustness` | candidate orders of one pairwise judge | reversal rate and position bias |
+| configuration change | two configurations, via `study.compare(a=..., b=...)` | paired interval on the difference; the model and trace fields that changed |
+
+The single-configuration designs (repeatability, prompt robustness, order robustness) can hold several configurations to compare, such as an old and a new judge. Each configuration is checked on its own. `alpha()` refuses to pool them; `study.compare(a={"condition_id": "old"}, b={"condition_id": "new"})` reports each one's alpha and diagnostics, with a paired interval on the difference.
+
+Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](examples/llm_judge.py).
+
+## Licence
+
+BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.

reliability_lab-0.1.0/README.md

@@ -0,0 +1,148 @@
+# reliability-lab
+
+**Status: alpha (0.1.0).** The API may still change between releases.
+
+reliability-lab is a study-design library for measuring the reliability of LLM and human judgments. It is built on Krippendorff's alpha. Alpha implementations already exist in Python. This package adds the workflow around the coefficient:
+
+- a validated long-form ratings schema in which every rating has an explicit status (`observed`, `abstained`, `inapplicable`, `parse_error`, `provider_error`);
+- declared measurement semantics: level, value domain, coder definition and design label;
+- white-box output: observed and expected disagreement, coincidence matrices and per-unit contributions;
+- explicit missing-data policies, because LLM failures are rarely unrelated to the item;
+- bootstrap intervals by a scheme you name: whole units or groups resampled (`scheme="units"`), or Krippendorff's KALPHA algorithm (`scheme="kalpha"`);
+- paired comparisons of two configurations on the same units, with the interval on the difference;
+- group-safe train/test splits that work with any scikit-learn splitter, with a saved manifest that reproduces the split;
+- versioned JSON and Parquet reports with provenance;
+- LLM judge workflows: a trace sidecar of judge configurations, declared retry handling, and evidence specific to each design (run agreement, order swaps and position bias).
+
+Mixed human-LLM panels and coverage simulations for the bootstrap schemes are next on the roadmap ([design document](docs/design/framing-v2.md)).
+
+> **Reliability is not validity.** Coders can agree consistently and still be wrong in the same way, for example because of a flawed rubric or a bias they share. When results come from different designs (repeatability, inter-model agreement, human panels), they are reported as different quantities and are never pooled.
+
+## Install
+
+```bash
+pip install "reliability-lab[sklearn,parquet]"
+```
+
+The extras are optional. `sklearn` provides the splitters, although any object with a scikit-learn style `split` method works; `parquet` enables `to_parquet`. For development, run `uv sync` and then `uv run pytest`.
+
+## Example
+
+```python
+import pandas as pd
+from reliability_lab import CoderSpec, RatingFrame, ReliabilityStudy
+
+df = pd.DataFrame(
+    {
+        "item": ["u1", "u1", "u2", "u2", "u3", "u3"],
+        "annotator": ["ann", "bob", "ann", "bob", "ann", "bob"],
+        "label": ["pos", "pos", "neg", "pos", "neg", "neg"],
+    }
+)
+
+ratings = RatingFrame.from_long(df, unit="item", rater="annotator", value="label")
+study = ReliabilityStudy(
+    ratings,
+    measurement="nominal",
+    values=["neg", "pos"],
+    coders=CoderSpec(by="rater_id"),
+    design="human_panel",
+)
+result = study.alpha()
+print(result.summary())
+```
+
+## Intervals and comparisons
+
+Every interval names its scheme, because the two schemes answer different questions. `scheme` is a required argument.
+
+```python
+boot = result.bootstrap(scheme="units", n_resamples=2000, random_state=7)
+print(boot.summary())  # 95% percentile intervals for every missing-data policy
+```
+
+- `scheme="units"` resamples whole units, and whole groups when a `group` column is declared. It can also resample within declared strata (`strata=True`), and it recomputes the expected disagreement in every replicate. `method="bca"` is available as an experimental option.
+- `scheme="kalpha"` reproduces Krippendorff's KALPHA algorithm. It resamples pairs of values and holds the expected disagreement fixed.
+
+To compare two configurations of a judge, for example two prompts, select each one's rows. Both must cover the same units. Every replicate resamples the same units for both, and the interval is on the difference:
+
+```python
+diff = study.compare(
+    a={"condition_id": "prompt_v1"},
+    b={"condition_id": "prompt_v2"},
+    n_resamples=2000,
+    random_state=7,
+)
+print(diff.summary())  # alpha_b - alpha_a with a paired interval
+```
+
+## Splits and reports
+
+Teams tune the judge (prompt, few-shot examples, rubric) and should report reliability on units the tuning never saw. So split before tuning, freeze the judge, then estimate on the test partition.
+
+When groups are declared, the splitter sees one sample per group (a conversation, say), so groups and repeated LLM calls never cross partitions:
+
+```python
+from sklearn.model_selection import StratifiedGroupKFold
+
+dev, test = study.holdout(
+    StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=7),
+    group_stratum="majority",  # needed only when a group's units span several strata
+)
+test.save_manifest("split.json")  # the ids of every partition, not just the seed
+
+# ... tune and freeze the judge on dev ...
+
+result = test.alpha()
+result.to_json("alpha_test.json")  # strict JSON with provenance
+result.to_parquet("alpha_test/")  # tidy tables
+
+dev2, test2 = later_study.apply_manifest("split.json")  # same units, new judge runs
+test.manifest.assert_no_leakage(few_shot_units)  # raises LeakageError
+```
+
+See [`examples/splits.py`](examples/splits.py) for a full run. Releases are described in [docs/releasing.md](docs/releasing.md).
+
+## LLM judge workflows
+
+A `condition_id` names a frozen judge configuration. A trace sidecar records what each configuration was: provider, prompt hash, rubric and parser versions, temperature, seed, and candidate order for pairwise judges. With it, a study checks that its coders differ only in what the design compares. For example, if prompt-robustness conditions also differ in temperature, it raises a `TraceWarning`. Whatever the trace leaves out is listed among the provenance gaps.
+
+```python
+from reliability_lab import TraceSidecar
+
+trace = TraceSidecar.from_frame(
+    configs,
+    condition="condition_id",
+    prompt_hash="prompt_sha",
+    temperature="temperature",
+    candidate_order="order",  # "A,B" / "B,A"
+)
+study = ReliabilityStudy(
+    ratings,  # RatingFrame.from_long(..., attempt="attempt")
+    measurement="nominal",
+    values=["A", "tie", "B"],  # labels name the winning candidate, not its position
+    coders="condition_id",
+    design="order_robustness",
+    trace=trace,
+    retries="last",  # required when calls were retried
+)
+result = study.alpha()
+result.diagnostics["order_swap"].values  # reversal rate, P(first shown wins) with a Wilson interval
+result.trace_findings  # confounds and caveats from the trace
+```
+
+| Design | Coders are | Evidence beside alpha |
+| --- | --- | --- |
+| `repeatability` | repeated runs of one configuration (`run_id`) | run agreement: units where every run agrees |
+| `prompt_robustness` | prompt conditions of one model | pairwise agreement between conditions |
+| `inter_model` | different models | pairwise agreement; family and configuration checks |
+| `order_robustness` | candidate orders of one pairwise judge | reversal rate and position bias |
+| configuration change | two configurations, via `study.compare(a=..., b=...)` | paired interval on the difference; the model and trace fields that changed |
+
+The single-configuration designs (repeatability, prompt robustness, order robustness) can hold several configurations to compare, such as an old and a new judge. Each configuration is checked on its own. `alpha()` refuses to pool them; `study.compare(a={"condition_id": "old"}, b={"condition_id": "new"})` reports each one's alpha and diagnostics, with a paired interval on the difference.
+
+Retried calls are never resolved silently. Number the attempts with an `attempt` column, then declare `retries="last"` or `retries="first_observed"`. Coverage then reports error rates both per attempt and for the attempts that count. See [`examples/llm_judge.py`](examples/llm_judge.py).
+
+## Licence
+
+BSD-3-Clause. Alpha is implemented independently from Krippendorff's papers. The GPL-licensed `krippendorff` package is used only in the test suite, as a parity check.