judgepanel 0.1.0__tar.gz

judgepanel-0.1.0/CITATION.cff

@@ -0,0 +1,30 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "judgepanel: estimating LLM-judge accuracy without gold labels"
+type: software
+version: "0.1.0"
+date-released: "2026-08-05"
+license: MIT
+repository-code: "https://github.com/mohammadi-hadi/judgepanel"
+url: "https://github.com/mohammadi-hadi/judgepanel"
+authors:
+  - given-names: "Hadi"
+    family-names: "Mohammadi"
+    orcid: "https://orcid.org/0000-0003-0860-9200"
+    affiliation: "Utrecht University"
+keywords:
+  - "LLM-as-judge"
+  - "Dawid-Skene"
+  - "label aggregation"
+  - "inter-rater agreement"
+  - "evaluation"
+  - "large language models"
+abstract: >-
+  Estimates each judge's confusion matrix, the class prevalence, and a
+  posterior over every item's true label from a panel of imperfect judges
+  labelling the same items, with no gold labels, via the Dawid-Skene model
+  fitted by EM. Includes majority-vote baselines, Cohen's and Fleiss' kappa,
+  bootstrap confidence intervals, and panel simulators. Validated on a real
+  five-judge panel over 400 agent trajectories where the estimated judge
+  accuracies can be checked against construction-time gold labels, including
+  an honest account of the correlated-judge failure mode.

judgepanel-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hadi Mohammadi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

judgepanel-0.1.0/PKG-INFO

@@ -0,0 +1,171 @@
+Metadata-Version: 2.4
+Name: judgepanel
+Version: 0.1.0
+Summary: Estimate LLM-judge accuracy without gold labels and aggregate judge panels: Dawid-Skene EM, agreement statistics, bootstrap uncertainty
+Project-URL: Repository, https://github.com/mohammadi-hadi/judgepanel
+Project-URL: Issues, https://github.com/mohammadi-hadi/judgepanel/issues
+Author: Hadi Mohammadi
+License-Expression: MIT
+License-File: LICENSE
+Keywords: annotation,dawid-skene,evaluation,inter-rater-agreement,label-aggregation,llm-as-judge
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Requires-Dist: numpy>=1.26
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Requires-Dist: twine>=6.1; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# judgepanel
+
+[![CI](https://github.com/mohammadi-hadi/judgepanel/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/judgepanel/actions/workflows/ci.yml)
+
+**Estimate how accurate your LLM judges are — without gold labels — and aggregate
+their verdicts better than majority vote.**
+
+When several imperfect judges label the same items, the disagreement pattern
+itself reveals who is reliable. judgepanel fits the classic Dawid–Skene model
+(1979) to a judge panel and recovers, jointly:
+
+- each judge's **confusion matrix** (sensitivity/specificity in the binary case),
+- the **class prevalence**,
+- a **posterior over every item's true label**, weighted by estimated judge
+  quality instead of one-judge-one-vote.
+
+Around the core: majority-vote baselines with explicit tie handling,
+chance-corrected agreement (Cohen's and Fleiss' kappa), bootstrap confidence
+intervals, and simulators for planning panels and studying failure modes.
+Pure NumPy; no model calls; every fit deterministic.
+
+## Install
+
+```bash
+pip install git+https://github.com/mohammadi-hadi/judgepanel
+```
+
+## Sixty seconds
+
+Your data is one JSON object per (item, judge) verdict:
+
+```json
+{"item": "case-0017", "judge": "gpt-strict", "label": "faulty"}
+```
+
+```bash
+judgepanel fit examples/data/demo_panel.jsonl --positive faulty --bootstrap 200
+```
+
+```
+panel: 300 items, 5 judges, 2 classes (clean, faulty)
+converged after 55 iterations
+estimated prevalence: clean=0.577, faulty=0.423
+
+judge            sens    spec  youden
+gpt-strict      0.906   0.931   0.836
+gpt-lenient     0.715   0.999   0.714
+small-model     0.692   0.710   0.402
+rules           0.562   0.996   0.559
+flag-happy      0.953   0.394   0.347
+```
+
+The demo panel is synthetic with known truth (`examples/data/demo_gold.jsonl`):
+the true parameters are (0.92, 0.94), (0.70, 0.97), (0.66, 0.72), (0.55, 0.99)
+and (0.97, 0.35) — every estimate above lands within 0.04 without ever seeing a
+gold label, and the flag-everything judge is exposed immediately.
+
+Same thing in Python:
+
+```python
+from judgepanel import fit_dawid_skene, read_panel_jsonl
+
+panel = read_panel_jsonl("panel.jsonl")
+result = fit_dawid_skene(panel)
+result.sensitivity("faulty")   # {"gpt-strict": 0.906, ...}
+result.labels()                # posterior-aggregated label per item
+```
+
+Keys are configurable, so existing verdict files work directly
+(`--item-key trajectory_id --judge-key judge_id --label-key faulty`; boolean
+labels become `"true"`/`"false"`).
+
+## Validation on a real judge panel
+
+The [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
+study had five LLM judges label the same 400 agent trajectories as faulty or
+clean, with gold labels known by construction. Fitting judgepanel on the
+verdicts alone (gold hidden), then unblinding
+(`python examples/case_study.py --raw .../results/raw`):
+
+| judge | est sens | true sens | est spec | true spec |
+|---|---|---|---|---|
+| programmatic rules | 0.771 | 0.667 | 0.993 | 1.000 |
+| outcome-only (14B) | 0.639 | 0.613 | 0.633 | 0.670 |
+| step-rubric (14B) | 0.995 | 0.857 | 1.000 | 1.000 |
+| step-rubric (8B) | 0.988 | 0.990 | 0.000 | 0.000 |
+| self-consistency k=3 (14B) | 0.980 | 0.843 | 0.994 | 0.990 |
+
+Three honest findings:
+
+1. **The spammer is exposed exactly.** The 8B judge flags nearly everything;
+   its estimated specificity is 0.000 — matching the truth — with no gold
+   label in sight. The judge quality *ranking* is right across the board, and
+   the production-default outcome judge is estimated within 0.04.
+2. **Correlated judges inflate their own estimates.** Self-consistency is the
+   step judge sampled three times, and the two agree at kappa 0.96. Dawid–Skene
+   assumes conditionally independent judges, so this pair drags the latent
+   truth toward its shared verdicts: both get overestimated by up to +0.14,
+   and estimated prevalence drops to 0.645 against a true 0.750. The library
+   prints the kappa matrix precisely so you check this before believing the fit.
+3. **No aggregation recovers what every judge misses.** Faults that leave the
+   final answer correct are invisible to the whole panel, so posterior
+   aggregation (0.895 accuracy) cannot beat majority (0.897) here. Weighted
+   aggregation wins when judge quality is *uneven*, not when errors are shared.
+
+## When not to trust it
+
+- **Fewer than 3 judges**: weakly identified; the fit warns.
+- **Highly correlated judges** (same base model, ensembles of one judge,
+  shared blind spots): estimates bias toward the correlated cluster — check
+  `judgepanel agree` first, and prefer diverse judge families.
+- **A panel that is wrong on average**: if most judges are worse than chance,
+  the labels-and-mirror solution is indistinguishable without gold.
+- **Tiny or extremely imbalanced panels**: run `judgepanel simulate` with your
+  expected operating points to see the estimation error you should expect
+  before you rely on the numbers.
+
+## API
+
+| call | what it does |
+|---|---|
+| `read_panel_jsonl(path, ...)` | long-format JSONL → `Panel` |
+| `fit_dawid_skene(panel)` | EM fit → confusions, prevalence, posteriors |
+| `bootstrap_judges(panel, n_boot=200)` | percentile CIs for judge parameters |
+| `majority_vote(panel)` | baseline aggregation, ties made explicit |
+| `cohen_kappa`, `fleiss_kappa`, `pairwise_cohen_kappa` | agreement diagnostics |
+| `simulate_panel`, `simulate_confusion_panel` | synthetic panels with known truth |
+
+CLI: `judgepanel fit`, `judgepanel agree`, `judgepanel simulate`,
+`judgepanel version` (each supports `--json`).
+
+## Development
+
+```bash
+python -m venv .venv && .venv/bin/pip install -e ".[dev]"
+.venv/bin/pytest          # 52 tests: hand-computed values, seeded recovery, CLI
+.venv/bin/ruff check src tests && .venv/bin/mypy src
+```
+
+The suite checks statistical behaviour, not just plumbing: kappa against
+hand-computed tables, EM recovery on seeded synthetic panels, log-likelihood
+monotonicity, bootstrap coverage, and determinism.
+
+## Citation
+
+If you use judgepanel, please cite it (see `CITATION.cff`).
+
+MIT license.

judgepanel-0.1.0/README.md

@@ -0,0 +1,148 @@
+# judgepanel
+
+[![CI](https://github.com/mohammadi-hadi/judgepanel/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/judgepanel/actions/workflows/ci.yml)
+
+**Estimate how accurate your LLM judges are — without gold labels — and aggregate
+their verdicts better than majority vote.**
+
+When several imperfect judges label the same items, the disagreement pattern
+itself reveals who is reliable. judgepanel fits the classic Dawid–Skene model
+(1979) to a judge panel and recovers, jointly:
+
+- each judge's **confusion matrix** (sensitivity/specificity in the binary case),
+- the **class prevalence**,
+- a **posterior over every item's true label**, weighted by estimated judge
+  quality instead of one-judge-one-vote.
+
+Around the core: majority-vote baselines with explicit tie handling,
+chance-corrected agreement (Cohen's and Fleiss' kappa), bootstrap confidence
+intervals, and simulators for planning panels and studying failure modes.
+Pure NumPy; no model calls; every fit deterministic.
+
+## Install
+
+```bash
+pip install git+https://github.com/mohammadi-hadi/judgepanel
+```
+
+## Sixty seconds
+
+Your data is one JSON object per (item, judge) verdict:
+
+```json
+{"item": "case-0017", "judge": "gpt-strict", "label": "faulty"}
+```
+
+```bash
+judgepanel fit examples/data/demo_panel.jsonl --positive faulty --bootstrap 200
+```
+
+```
+panel: 300 items, 5 judges, 2 classes (clean, faulty)
+converged after 55 iterations
+estimated prevalence: clean=0.577, faulty=0.423
+
+judge            sens    spec  youden
+gpt-strict      0.906   0.931   0.836
+gpt-lenient     0.715   0.999   0.714
+small-model     0.692   0.710   0.402
+rules           0.562   0.996   0.559
+flag-happy      0.953   0.394   0.347
+```
+
+The demo panel is synthetic with known truth (`examples/data/demo_gold.jsonl`):
+the true parameters are (0.92, 0.94), (0.70, 0.97), (0.66, 0.72), (0.55, 0.99)
+and (0.97, 0.35) — every estimate above lands within 0.04 without ever seeing a
+gold label, and the flag-everything judge is exposed immediately.
+
+Same thing in Python:
+
+```python
+from judgepanel import fit_dawid_skene, read_panel_jsonl
+
+panel = read_panel_jsonl("panel.jsonl")
+result = fit_dawid_skene(panel)
+result.sensitivity("faulty")   # {"gpt-strict": 0.906, ...}
+result.labels()                # posterior-aggregated label per item
+```
+
+Keys are configurable, so existing verdict files work directly
+(`--item-key trajectory_id --judge-key judge_id --label-key faulty`; boolean
+labels become `"true"`/`"false"`).
+
+## Validation on a real judge panel
+
+The [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
+study had five LLM judges label the same 400 agent trajectories as faulty or
+clean, with gold labels known by construction. Fitting judgepanel on the
+verdicts alone (gold hidden), then unblinding
+(`python examples/case_study.py --raw .../results/raw`):
+
+| judge | est sens | true sens | est spec | true spec |
+|---|---|---|---|---|
+| programmatic rules | 0.771 | 0.667 | 0.993 | 1.000 |
+| outcome-only (14B) | 0.639 | 0.613 | 0.633 | 0.670 |
+| step-rubric (14B) | 0.995 | 0.857 | 1.000 | 1.000 |
+| step-rubric (8B) | 0.988 | 0.990 | 0.000 | 0.000 |
+| self-consistency k=3 (14B) | 0.980 | 0.843 | 0.994 | 0.990 |
+
+Three honest findings:
+
+1. **The spammer is exposed exactly.** The 8B judge flags nearly everything;
+   its estimated specificity is 0.000 — matching the truth — with no gold
+   label in sight. The judge quality *ranking* is right across the board, and
+   the production-default outcome judge is estimated within 0.04.
+2. **Correlated judges inflate their own estimates.** Self-consistency is the
+   step judge sampled three times, and the two agree at kappa 0.96. Dawid–Skene
+   assumes conditionally independent judges, so this pair drags the latent
+   truth toward its shared verdicts: both get overestimated by up to +0.14,
+   and estimated prevalence drops to 0.645 against a true 0.750. The library
+   prints the kappa matrix precisely so you check this before believing the fit.
+3. **No aggregation recovers what every judge misses.** Faults that leave the
+   final answer correct are invisible to the whole panel, so posterior
+   aggregation (0.895 accuracy) cannot beat majority (0.897) here. Weighted
+   aggregation wins when judge quality is *uneven*, not when errors are shared.
+
+## When not to trust it
+
+- **Fewer than 3 judges**: weakly identified; the fit warns.
+- **Highly correlated judges** (same base model, ensembles of one judge,
+  shared blind spots): estimates bias toward the correlated cluster — check
+  `judgepanel agree` first, and prefer diverse judge families.
+- **A panel that is wrong on average**: if most judges are worse than chance,
+  the labels-and-mirror solution is indistinguishable without gold.
+- **Tiny or extremely imbalanced panels**: run `judgepanel simulate` with your
+  expected operating points to see the estimation error you should expect
+  before you rely on the numbers.
+
+## API
+
+| call | what it does |
+|---|---|
+| `read_panel_jsonl(path, ...)` | long-format JSONL → `Panel` |
+| `fit_dawid_skene(panel)` | EM fit → confusions, prevalence, posteriors |
+| `bootstrap_judges(panel, n_boot=200)` | percentile CIs for judge parameters |
+| `majority_vote(panel)` | baseline aggregation, ties made explicit |
+| `cohen_kappa`, `fleiss_kappa`, `pairwise_cohen_kappa` | agreement diagnostics |
+| `simulate_panel`, `simulate_confusion_panel` | synthetic panels with known truth |
+
+CLI: `judgepanel fit`, `judgepanel agree`, `judgepanel simulate`,
+`judgepanel version` (each supports `--json`).
+
+## Development
+
+```bash
+python -m venv .venv && .venv/bin/pip install -e ".[dev]"
+.venv/bin/pytest          # 52 tests: hand-computed values, seeded recovery, CLI
+.venv/bin/ruff check src tests && .venv/bin/mypy src
+```
+
+The suite checks statistical behaviour, not just plumbing: kappa against
+hand-computed tables, EM recovery on seeded synthetic panels, log-likelihood
+monotonicity, bootstrap coverage, and determinism.
+
+## Citation
+
+If you use judgepanel, please cite it (see `CITATION.cff`).
+
+MIT license.

judgepanel-0.1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "judgepanel"
+version = "0.1.0"
+description = "Estimate LLM-judge accuracy without gold labels and aggregate judge panels: Dawid-Skene EM, agreement statistics, bootstrap uncertainty"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Hadi Mohammadi" }]
+requires-python = ">=3.12"
+keywords = [
+  "llm-as-judge",
+  "dawid-skene",
+  "label-aggregation",
+  "inter-rater-agreement",
+  "annotation",
+  "evaluation",
+]
+classifiers = [
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+dependencies = ["numpy>=1.26"]
+
+[project.urls]
+Repository = "https://github.com/mohammadi-hadi/judgepanel"
+Issues = "https://github.com/mohammadi-hadi/judgepanel/issues"
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0",
+  "ruff>=0.5",
+  "mypy>=1.10",
+  "build>=1.2",
+  "twine>=6.1",
+]
+
+[project.scripts]
+judgepanel = "judgepanel.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/judgepanel"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE", "CITATION.cff"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+warn_unreachable = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

judgepanel-0.1.0/src/judgepanel/__init__.py

@@ -0,0 +1,39 @@
+"""Estimate LLM-judge accuracy without gold labels and aggregate judge panels.
+
+The core is the Dawid-Skene model: several imperfect judges label the same
+items, and EM jointly recovers each judge's confusion matrix, the class
+prevalence, and a posterior over every item's true label - no gold labels
+required. Around it: majority-vote baselines, chance-corrected agreement,
+bootstrap confidence intervals, and simulators for planning panels and
+studying failure modes such as correlated judges.
+"""
+
+from judgepanel.agreement import cohen_kappa, fleiss_kappa, pairwise_cohen_kappa
+from judgepanel.em import DawidSkeneResult, fit_dawid_skene
+from judgepanel.io import read_panel_jsonl, write_labels_jsonl
+from judgepanel.panel import MISSING, Panel
+from judgepanel.simulate import SimJudge, simulate_confusion_panel, simulate_panel
+from judgepanel.uncertainty import BootstrapResult, bootstrap_judges
+from judgepanel.vote import MajorityResult, majority_vote
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MISSING",
+    "BootstrapResult",
+    "DawidSkeneResult",
+    "MajorityResult",
+    "Panel",
+    "SimJudge",
+    "__version__",
+    "bootstrap_judges",
+    "cohen_kappa",
+    "fit_dawid_skene",
+    "fleiss_kappa",
+    "majority_vote",
+    "pairwise_cohen_kappa",
+    "read_panel_jsonl",
+    "simulate_confusion_panel",
+    "simulate_panel",
+    "write_labels_jsonl",
+]

judgepanel-0.1.0/src/judgepanel/agreement.py

@@ -0,0 +1,80 @@
+"""Chance-corrected agreement between judges."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+import numpy.typing as npt
+
+from judgepanel.panel import MISSING, Panel
+
+FloatArray = npt.NDArray[np.float64]
+
+_EPS = 1e-12
+
+
+def cohen_kappa(a: Sequence[object], b: Sequence[object]) -> float:
+    """Cohen's kappa between two aligned label sequences.
+
+    Returns 1.0 when both raters agree everywhere with no variation, and NaN
+    when chance agreement is 1 but observed agreement is not (kappa undefined).
+    """
+    if len(a) != len(b):
+        raise ValueError("sequences must be aligned")
+    if not a:
+        raise ValueError("no labels")
+    labels_a = list(a)
+    labels_b = list(b)
+    classes = sorted({*labels_a, *labels_b}, key=repr)
+    index = {c: k for k, c in enumerate(classes)}
+    n = len(labels_a)
+    table = np.zeros((len(classes), len(classes)), dtype=np.float64)
+    for x, y in zip(labels_a, labels_b, strict=True):
+        table[index[x], index[y]] += 1.0
+    table /= n
+    observed = float(np.trace(table))
+    expected = float(table.sum(axis=1) @ table.sum(axis=0))
+    if 1.0 - expected < _EPS:
+        return 1.0 if 1.0 - observed < _EPS else float("nan")
+    return (observed - expected) / (1.0 - expected)
+
+
+def pairwise_cohen_kappa(panel: Panel) -> FloatArray:
+    """Cohen's kappa for every judge pair, on their co-observed items.
+
+    The diagonal is 1. A pair with fewer than 2 co-observed items is NaN.
+    """
+    out = np.full((panel.n_judges, panel.n_judges), np.nan, dtype=np.float64)
+    np.fill_diagonal(out, 1.0)
+    labels = panel.labels
+    for j in range(panel.n_judges):
+        for m in range(j + 1, panel.n_judges):
+            both = (labels[:, j] != MISSING) & (labels[:, m] != MISSING)
+            if int(both.sum()) < 2:
+                continue
+            kappa = cohen_kappa(labels[both, j].tolist(), labels[both, m].tolist())
+            out[j, m] = out[m, j] = kappa
+    return out
+
+
+def fleiss_kappa(panel: Panel) -> float:
+    """Fleiss' kappa over the panel, generalised to a variable rater count.
+
+    Items with fewer than 2 observed labels carry no pairwise-agreement
+    information and are skipped.
+    """
+    counts = panel.counts().astype(np.float64)
+    per_item = counts.sum(axis=1)
+    usable = per_item >= 2
+    if not bool(usable.any()):
+        raise ValueError("no item has 2 or more labels")
+    counts = counts[usable]
+    per_item = per_item[usable]
+    agree = ((counts * (counts - 1)).sum(axis=1)) / (per_item * (per_item - 1))
+    observed = float(agree.mean())
+    shares = counts.sum(axis=0) / counts.sum()
+    expected = float(shares @ shares)
+    if 1.0 - expected < _EPS:
+        return 1.0 if 1.0 - observed < _EPS else float("nan")
+    return (observed - expected) / (1.0 - expected)