reliagate 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Youssef Ouhaghi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,138 @@
1
+ Metadata-Version: 2.4
2
+ Name: reliagate
3
+ Version: 0.2.0
4
+ Summary: Your agent passed CI once. That proves nothing. Reliability metrics and statistically sound CI gates for AI agents.
5
+ Author: Youssef Ouhaghi
6
+ License-Expression: MIT
7
+ Project-URL: Repository, https://github.com/Gjusev/reliagate
8
+ Project-URL: Documentation, https://github.com/Gjusev/reliagate/tree/main/docs
9
+ Project-URL: Issues, https://github.com/Gjusev/reliagate/issues
10
+ Keywords: llm-agents,evaluation,statistics,pass-k,mcnemar,ci-gate
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Topic :: Software Development :: Testing
16
+ Classifier: Topic :: Scientific/Engineering
17
+ Requires-Python: >=3.10
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Dynamic: license-file
21
+
22
+ <p align="center">
23
+ <picture>
24
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/brand/lockup-dark.svg">
25
+ <img src="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/brand/lockup-light.svg" alt="reliagate" width="420">
26
+ </picture>
27
+ </p>
28
+
29
+ <p align="center">
30
+ <b>Your agent passed CI once. That proves nothing.</b><br>
31
+ Reliability metrics and statistically sound CI gates for AI agents evaluated with repeated runs.
32
+ </p>
33
+
34
+ <p align="center">
35
+ <a href="https://github.com/Gjusev/reliagate/actions/workflows/tests.yml"><img src="https://github.com/Gjusev/reliagate/actions/workflows/tests.yml/badge.svg" alt="tests"></a>
36
+ <img src="https://img.shields.io/badge/dependencies-0-2EBD78" alt="zero dependencies">
37
+ <img src="https://img.shields.io/badge/python-3.10%2B-2EBD78" alt="python 3.10+">
38
+ <a href="https://github.com/Gjusev/reliagate/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-2EBD78" alt="MIT license"></a>
39
+ </p>
40
+
41
+ <p align="center">
42
+ <a href="https://github.com/Gjusev/reliagate/blob/main/assets/video/reliagate.mp4"><img src="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/video/preview.gif" alt="reliagate in 30 seconds: one run passes, four runs don't; pass@4 stays 1.00 while pass^4 drops 0.85 to 0.65; the gate exits 2" width="760"></a>
43
+ <br>
44
+ <sub><a href="https://github.com/Gjusev/reliagate/blob/main/assets/video/reliagate.mp4">Watch the 30-second film</a>. Every number in it is real output of this repo's <a href="https://github.com/Gjusev/reliagate/blob/main/examples/">example files</a>.</sub>
45
+ </p>
46
+
47
+ ---
48
+
49
+ Most agent evals run each task **once** and gate CI on a score threshold. Agents are stochastic, so the same commit can pass on Monday and fail on Tuesday, and you get false alarms and false all-clears.
50
+
51
+ **reliagate** treats the *task* as the unit of evidence:
52
+
53
+ 1. **Run every task `k` times** and ask whether it is solved *consistently*, not just once.
54
+ 2. **Measure reliability**, not luck: `pass^k`, unbiased `pass@k`, consistency classes, Wilson and task-level bootstrap intervals.
55
+ 3. **Gate on a paired statistical test** between two versions (McNemar, with Wilcoxon and a Newcombe interval alongside), with exit codes your CI can trust.
56
+
57
+ Pure Python, **zero dependencies**, 148 offline tests.
58
+
59
+ ## Why pass@k is not enough
60
+
61
+ The film above uses the two files in [`examples/`](examples/): the same 40 tasks, 4 runs each, before and after a change.
62
+
63
+ | | v1 | v2 |
64
+ |---|---|---|
65
+ | `pass@4`: solved at least once | **1.00** | **1.00** |
66
+ | `pass^4`: solved every time | **0.85** | **0.65** |
67
+
68
+ `pass@4` says nothing changed. In fact 9 tasks that used to pass every run now fail some of the time, and only 1 went the other way:
69
+
70
+ ```console
71
+ $ reliagate gate --base examples/base.jsonl --new examples/new.jsonl
72
+ gate decision: REGRESSION (exit 2)
73
+ consistently solved: base 34/40 -> new 26/40
74
+ gained (b)=1 lost (c)=9 b-c=-8 test=mcnemar_exact p=0.02148
75
+ ...
76
+ ```
77
+
78
+ ## Quickstart
79
+
80
+ ```bash
81
+ pip install reliagate
82
+ ```
83
+
84
+ ```python
85
+ from reliagate import read_jsonl, pass_k, pass_at_k, compare, gate
86
+
87
+ v1 = read_jsonl("examples/base.jsonl") # one line per trial: {"task_id", "attempt", "success"}
88
+ v2 = read_jsonl("examples/new.jsonl")
89
+
90
+ pass_at_k(v2, k=4) # 1.0 solved at least once in 4
91
+ pass_k(v2, k=4).value # 0.65 solved in all 4
92
+ pass_k(v2, k=4, ci="wilson_indicator").ci # (0.495, 0.779)
93
+
94
+ r = compare(v1, v2) # paired McNemar on "consistently solved"
95
+ r.b, r.c, r.p_value # (1, 9, 0.02148)
96
+
97
+ gate(v1, v2, max_regression=0.05).exit_code # 2: significant regression
98
+ ```
99
+
100
+ Trials are `{"task_id": "checkout", "attempt": 0, "success": true}`. `success: null` means the trial abstained or hit an infrastructure error. It stays in the denominator and is **never** counted as a failure.
101
+
102
+ ## Gate your CI
103
+
104
+ ```yaml
105
+ - run: pip install reliagate
106
+ - run: reliagate gate --base evals/main.jsonl --new evals/pr.jsonl --max-regression 0.05
107
+ ```
108
+
109
+ | Exit | Meaning |
110
+ |---|---|
111
+ | `0` | pass: the drop in consistently-solved tasks is within `--max-regression` |
112
+ | `2` | significant regression in consistently-solved tasks |
113
+ | `3` | infra or usage error, including too many infra-errored trials: never a silent pass |
114
+ | `4` | inconclusive: a drop beyond the limit that the data can't confirm. Run more tasks |
115
+
116
+ Full recipe, including how to size the run with `plan()` and stop early with SPRT: [docs/ci-gate.md](docs/ci-gate.md).
117
+
118
+ ## Documentation
119
+
120
+ | | |
121
+ |---|---|
122
+ | [Getting started](docs/getting-started.md) | data format, the example walkthrough, collecting runs |
123
+ | [Metrics](docs/metrics.md) | every estimator, its definition and what it is verified against |
124
+ | [CI gate](docs/ci-gate.md) | GitHub Actions recipe, exit codes, planning and early stopping |
125
+ | [Design and limitations](docs/design.md) | abstentions, zero dependencies, what reliagate does not do |
126
+ | [First real study](docs/results/first-study.md) | the toolkit on 494 real tau-bench retail tasks |
127
+ | [Changelog](CHANGELOG.md) | what changed in each release |
128
+ | [Roadmap](ROADMAP.md) | what's done and what's next |
129
+
130
+ ## Validated on real data
131
+
132
+ [docs/results/first-study.md](docs/results/first-study.md) applies reliagate to real tau-bench retail trajectories, regenerated deterministically by `scripts/first_study.py` with no LLM calls. On 494 tasks at k=2, **pass^2 = 0.907** (Wilson 95% CI [0.878, 0.930]). With k=4, the pass@k minus pass^k gap reaches **+0.20** for one gating policy. All five paired policy comparisons come out null and underpowered, and the study says so instead of rounding them into a win.
133
+
134
+ ## Contributing
135
+
136
+ Issues and PRs are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md) first: the core stays stdlib-only and every claim must be backed by a test or a rerunnable script. If you use reliagate in research, please cite it ([CITATION.cff](CITATION.cff)).
137
+
138
+ MIT © 2026 Youssef Ouhaghi
@@ -0,0 +1,117 @@
1
+ <p align="center">
2
+ <picture>
3
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/brand/lockup-dark.svg">
4
+ <img src="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/brand/lockup-light.svg" alt="reliagate" width="420">
5
+ </picture>
6
+ </p>
7
+
8
+ <p align="center">
9
+ <b>Your agent passed CI once. That proves nothing.</b><br>
10
+ Reliability metrics and statistically sound CI gates for AI agents evaluated with repeated runs.
11
+ </p>
12
+
13
+ <p align="center">
14
+ <a href="https://github.com/Gjusev/reliagate/actions/workflows/tests.yml"><img src="https://github.com/Gjusev/reliagate/actions/workflows/tests.yml/badge.svg" alt="tests"></a>
15
+ <img src="https://img.shields.io/badge/dependencies-0-2EBD78" alt="zero dependencies">
16
+ <img src="https://img.shields.io/badge/python-3.10%2B-2EBD78" alt="python 3.10+">
17
+ <a href="https://github.com/Gjusev/reliagate/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-2EBD78" alt="MIT license"></a>
18
+ </p>
19
+
20
+ <p align="center">
21
+ <a href="https://github.com/Gjusev/reliagate/blob/main/assets/video/reliagate.mp4"><img src="https://raw.githubusercontent.com/Gjusev/reliagate/main/assets/video/preview.gif" alt="reliagate in 30 seconds: one run passes, four runs don't; pass@4 stays 1.00 while pass^4 drops 0.85 to 0.65; the gate exits 2" width="760"></a>
22
+ <br>
23
+ <sub><a href="https://github.com/Gjusev/reliagate/blob/main/assets/video/reliagate.mp4">Watch the 30-second film</a>. Every number in it is real output of this repo's <a href="https://github.com/Gjusev/reliagate/blob/main/examples/">example files</a>.</sub>
24
+ </p>
25
+
26
+ ---
27
+
28
+ Most agent evals run each task **once** and gate CI on a score threshold. Agents are stochastic, so the same commit can pass on Monday and fail on Tuesday, and you get false alarms and false all-clears.
29
+
30
+ **reliagate** treats the *task* as the unit of evidence:
31
+
32
+ 1. **Run every task `k` times** and ask whether it is solved *consistently*, not just once.
33
+ 2. **Measure reliability**, not luck: `pass^k`, unbiased `pass@k`, consistency classes, Wilson and task-level bootstrap intervals.
34
+ 3. **Gate on a paired statistical test** between two versions (McNemar, with Wilcoxon and a Newcombe interval alongside), with exit codes your CI can trust.
35
+
36
+ Pure Python, **zero dependencies**, 148 offline tests.
37
+
38
+ ## Why pass@k is not enough
39
+
40
+ The film above uses the two files in [`examples/`](examples/): the same 40 tasks, 4 runs each, before and after a change.
41
+
42
+ | | v1 | v2 |
43
+ |---|---|---|
44
+ | `pass@4`: solved at least once | **1.00** | **1.00** |
45
+ | `pass^4`: solved every time | **0.85** | **0.65** |
46
+
47
+ `pass@4` says nothing changed. In fact 9 tasks that used to pass every run now fail some of the time, and only 1 went the other way:
48
+
49
+ ```console
50
+ $ reliagate gate --base examples/base.jsonl --new examples/new.jsonl
51
+ gate decision: REGRESSION (exit 2)
52
+ consistently solved: base 34/40 -> new 26/40
53
+ gained (b)=1 lost (c)=9 b-c=-8 test=mcnemar_exact p=0.02148
54
+ ...
55
+ ```
56
+
57
+ ## Quickstart
58
+
59
+ ```bash
60
+ pip install reliagate
61
+ ```
62
+
63
+ ```python
64
+ from reliagate import read_jsonl, pass_k, pass_at_k, compare, gate
65
+
66
+ v1 = read_jsonl("examples/base.jsonl") # one line per trial: {"task_id", "attempt", "success"}
67
+ v2 = read_jsonl("examples/new.jsonl")
68
+
69
+ pass_at_k(v2, k=4) # 1.0 solved at least once in 4
70
+ pass_k(v2, k=4).value # 0.65 solved in all 4
71
+ pass_k(v2, k=4, ci="wilson_indicator").ci # (0.495, 0.779)
72
+
73
+ r = compare(v1, v2) # paired McNemar on "consistently solved"
74
+ r.b, r.c, r.p_value # (1, 9, 0.02148)
75
+
76
+ gate(v1, v2, max_regression=0.05).exit_code # 2: significant regression
77
+ ```
78
+
79
+ Trials are `{"task_id": "checkout", "attempt": 0, "success": true}`. `success: null` means the trial abstained or hit an infrastructure error. It stays in the denominator and is **never** counted as a failure.
80
+
81
+ ## Gate your CI
82
+
83
+ ```yaml
84
+ - run: pip install reliagate
85
+ - run: reliagate gate --base evals/main.jsonl --new evals/pr.jsonl --max-regression 0.05
86
+ ```
87
+
88
+ | Exit | Meaning |
89
+ |---|---|
90
+ | `0` | pass: the drop in consistently-solved tasks is within `--max-regression` |
91
+ | `2` | significant regression in consistently-solved tasks |
92
+ | `3` | infra or usage error, including too many infra-errored trials: never a silent pass |
93
+ | `4` | inconclusive: a drop beyond the limit that the data can't confirm. Run more tasks |
94
+
95
+ Full recipe, including how to size the run with `plan()` and stop early with SPRT: [docs/ci-gate.md](docs/ci-gate.md).
96
+
97
+ ## Documentation
98
+
99
+ | | |
100
+ |---|---|
101
+ | [Getting started](docs/getting-started.md) | data format, the example walkthrough, collecting runs |
102
+ | [Metrics](docs/metrics.md) | every estimator, its definition and what it is verified against |
103
+ | [CI gate](docs/ci-gate.md) | GitHub Actions recipe, exit codes, planning and early stopping |
104
+ | [Design and limitations](docs/design.md) | abstentions, zero dependencies, what reliagate does not do |
105
+ | [First real study](docs/results/first-study.md) | the toolkit on 494 real tau-bench retail tasks |
106
+ | [Changelog](CHANGELOG.md) | what changed in each release |
107
+ | [Roadmap](ROADMAP.md) | what's done and what's next |
108
+
109
+ ## Validated on real data
110
+
111
+ [docs/results/first-study.md](docs/results/first-study.md) applies reliagate to real tau-bench retail trajectories, regenerated deterministically by `scripts/first_study.py` with no LLM calls. On 494 tasks at k=2, **pass^2 = 0.907** (Wilson 95% CI [0.878, 0.930]). With k=4, the pass@k minus pass^k gap reaches **+0.20** for one gating policy. All five paired policy comparisons come out null and underpowered, and the study says so instead of rounding them into a win.
112
+
113
+ ## Contributing
114
+
115
+ Issues and PRs are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md) first: the core stays stdlib-only and every claim must be backed by a test or a rerunnable script. If you use reliagate in research, please cite it ([CITATION.cff](CITATION.cff)).
116
+
117
+ MIT © 2026 Youssef Ouhaghi
@@ -0,0 +1,38 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "reliagate"
7
+ version = "0.2.0"
8
+ description = "Your agent passed CI once. That proves nothing. Reliability metrics and statistically sound CI gates for AI agents."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Youssef Ouhaghi" }]
14
+ keywords = ["llm-agents", "evaluation", "statistics", "pass-k", "mcnemar", "ci-gate"]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Intended Audience :: Science/Research",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3 :: Only",
20
+ "Topic :: Software Development :: Testing",
21
+ "Topic :: Scientific/Engineering",
22
+ ]
23
+ # zero dependencies -- the core is pure stdlib by design
24
+ dependencies = []
25
+
26
+ [project.optional-dependencies]
27
+ # reserved for future lazy adapters (inspect-ai, langfuse); nothing yet
28
+
29
+ [project.scripts]
30
+ reliagate = "reliagate.cli:main"
31
+
32
+ [project.urls]
33
+ Repository = "https://github.com/Gjusev/reliagate"
34
+ Documentation = "https://github.com/Gjusev/reliagate/tree/main/docs"
35
+ Issues = "https://github.com/Gjusev/reliagate/issues"
36
+
37
+ [tool.setuptools.packages.find]
38
+ include = ["reliagate*"]
@@ -0,0 +1,103 @@
1
+ """reliagate -- reliability metrics and CI gates for AI agents."""
2
+
3
+ from .model import (
4
+ EvalRun,
5
+ Run,
6
+ abstention_report,
7
+ consistency_class,
8
+ consistency_classes,
9
+ group_by_task,
10
+ task_counts,
11
+ )
12
+ from .metrics import (
13
+ Estimate,
14
+ bootstrap_pass_pow_k,
15
+ pass_at_k,
16
+ pass_k,
17
+ pass_pow_k,
18
+ per_task,
19
+ consistency,
20
+ wilson_ci,
21
+ wilson_indicator,
22
+ )
23
+ from .compare import (
24
+ ComparisonResult,
25
+ compare,
26
+ mcnemar_exact,
27
+ mcnemar_midp,
28
+ )
29
+ from .stats import (
30
+ WilcoxonResult,
31
+ average_ranks,
32
+ benjamini_hochberg_adjust,
33
+ correct_pvalues,
34
+ hierarchical_bootstrap,
35
+ holm_adjust,
36
+ newcombe_paired,
37
+ wilcoxon_signed_rank,
38
+ )
39
+ from .planning import (
40
+ PlanResult,
41
+ SprtResult,
42
+ consistency_settled,
43
+ plan,
44
+ required_runs,
45
+ sprt_stop,
46
+ )
47
+ from .gate import (
48
+ EXIT_ERROR,
49
+ EXIT_INCONCLUSIVE,
50
+ EXIT_PASS,
51
+ EXIT_REGRESSION,
52
+ GateReport,
53
+ gate,
54
+ )
55
+ from .adapters.jsonl import JSONLError, read_jsonl, write_jsonl
56
+
57
+ __version__ = "0.2.0"
58
+
59
+ __all__ = [
60
+ "EvalRun",
61
+ "Run",
62
+ "abstention_report",
63
+ "consistency_class",
64
+ "consistency_classes",
65
+ "group_by_task",
66
+ "task_counts",
67
+ "Estimate",
68
+ "bootstrap_pass_pow_k",
69
+ "pass_at_k",
70
+ "pass_k",
71
+ "pass_pow_k",
72
+ "per_task",
73
+ "consistency",
74
+ "wilson_ci",
75
+ "wilson_indicator",
76
+ "ComparisonResult",
77
+ "compare",
78
+ "mcnemar_exact",
79
+ "mcnemar_midp",
80
+ "WilcoxonResult",
81
+ "average_ranks",
82
+ "benjamini_hochberg_adjust",
83
+ "correct_pvalues",
84
+ "hierarchical_bootstrap",
85
+ "holm_adjust",
86
+ "newcombe_paired",
87
+ "wilcoxon_signed_rank",
88
+ "PlanResult",
89
+ "SprtResult",
90
+ "consistency_settled",
91
+ "plan",
92
+ "required_runs",
93
+ "sprt_stop",
94
+ "EXIT_ERROR",
95
+ "EXIT_INCONCLUSIVE",
96
+ "EXIT_PASS",
97
+ "EXIT_REGRESSION",
98
+ "GateReport",
99
+ "gate",
100
+ "JSONLError",
101
+ "read_jsonl",
102
+ "write_jsonl",
103
+ ]
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
@@ -0,0 +1,5 @@
1
+ """Adapters: JSONL (always available), nitelos + Z.ai runner (opt-in).
2
+
3
+ Inspect AI and Langfuse adapters are documented stubs -- see ROADMAP.md.
4
+ Nothing here is imported by the statistics core.
5
+ """
@@ -0,0 +1,84 @@
1
+ """JSONL in/out adapter (stdlib only -- always available).
2
+
3
+ Line schema (one trial per line):
4
+ {"task_id": "checkout", "attempt": 0, "success": true,
5
+ "version": "abc123", "metadata": {"latency_ms": 4311}}
6
+ ``success`` is true, false or null (null = abstained/infra error).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from typing import Any
13
+
14
+ from ..model import EvalRun, Run
15
+
16
+
17
+ class JSONLError(ValueError):
18
+ """Raised on malformed JSONL -- the gate maps this to exit 3."""
19
+
20
+
21
+ def write_jsonl(run: EvalRun, path: str) -> int:
22
+ """Write an EvalRun to path; returns the number of lines."""
23
+ count = 0
24
+ with open(path, "w", encoding="utf-8") as fh:
25
+ for r in run.runs:
26
+ fh.write(json.dumps(r.to_dict() if hasattr(r, "to_dict") else {
27
+ "task_id": r.task_id,
28
+ "attempt": r.attempt,
29
+ "success": r.success,
30
+ "version": r.version,
31
+ "metadata": r.metadata,
32
+ }) + "\n")
33
+ count += 1
34
+ return count
35
+
36
+
37
+ def read_jsonl(path: str, name: str | None = None) -> EvalRun:
38
+ """Read a JSONL file of trials into an EvalRun.
39
+
40
+ Raises JSONLError (a ValueError) on unreadable files or malformed
41
+ lines -- callers must surface this as an infra error, never as a
42
+ comparison result.
43
+ """
44
+ runs: list[Run] = []
45
+ try:
46
+ with open(path, "r", encoding="utf-8") as fh:
47
+ lines = fh.readlines()
48
+ except OSError as exc:
49
+ raise JSONLError(f"cannot read {path!r}: {exc}") from exc
50
+
51
+ for lineno, raw in enumerate(lines, start=1):
52
+ line = raw.strip()
53
+ if not line:
54
+ continue
55
+ try:
56
+ obj: Any = json.loads(line)
57
+ except json.JSONDecodeError as exc:
58
+ raise JSONLError(f"{path}:{lineno}: not valid JSON: {exc}") from exc
59
+ if not isinstance(obj, dict):
60
+ raise JSONLError(f"{path}:{lineno}: expected a JSON object")
61
+ missing = [k for k in ("task_id", "attempt", "success") if k not in obj]
62
+ if missing:
63
+ raise JSONLError(
64
+ f"{path}:{lineno}: missing required keys {missing}"
65
+ )
66
+ metadata = obj.get("metadata", {})
67
+ if not isinstance(metadata, dict):
68
+ raise JSONLError(f"{path}:{lineno}: 'metadata' must be an object")
69
+ try:
70
+ runs.append(
71
+ Run(
72
+ task_id=obj["task_id"],
73
+ attempt=obj["attempt"],
74
+ success=obj["success"],
75
+ version=obj.get("version"),
76
+ metadata=metadata,
77
+ )
78
+ )
79
+ except (TypeError, ValueError) as exc:
80
+ raise JSONLError(f"{path}:{lineno}: {exc}") from exc
81
+
82
+ if not runs:
83
+ raise JSONLError(f"{path}: contains no trials")
84
+ return EvalRun(name=name or path, runs=runs)
@@ -0,0 +1,77 @@
1
+ """nitelos verdicts adapter (optional, lazy import).
2
+
3
+ Maps nitelos verdict objects to reliagate Run trials. nitelos is NOT a
4
+ dependency of the core; import this module only when nitelos is installed.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, Iterable
10
+
11
+ from ..model import Run
12
+
13
+
14
+ def _verdict_to_success(verdict: Any) -> bool | None:
15
+ """Map a nitelos verdict to True/False/None.
16
+
17
+ Known mappings (documented, best-effort -- nitelos is not pinned here):
18
+ * truthy string 'pass'/'passed'/'success'/'true' -> True
19
+ * 'fail'/'failed'/'false' -> False
20
+ * 'error'/'abstain'/'skipped'/None -> None (abstained, never a failure)
21
+ """
22
+ if verdict is None:
23
+ return None
24
+ if isinstance(verdict, bool):
25
+ return verdict
26
+ text = str(verdict).strip().lower()
27
+ if text in ("pass", "passed", "success", "true", "1"):
28
+ return True
29
+ if text in ("fail", "failed", "false", "0"):
30
+ return False
31
+ return None # error/abstain/unknown -> abstained
32
+
33
+
34
+ def verdicts_to_runs(
35
+ verdicts: Iterable[Any],
36
+ *,
37
+ task_ids: Iterable[str] | None = None,
38
+ version: str | None = None,
39
+ ) -> list[Run]:
40
+ """Convert an iterable of nitelos verdicts to reliagate Runs.
41
+
42
+ ``task_ids`` must align with ``verdicts`` when the verdicts do not
43
+ carry their own task identity (attribute ``task_id`` or mapping key).
44
+ """
45
+ task_list = list(task_ids) if task_ids is not None else None
46
+ runs: list[Run] = []
47
+ counters: dict[str, int] = {}
48
+ for i, verdict in enumerate(verdicts):
49
+ task_id: str | None = None
50
+ raw = verdict
51
+ if isinstance(verdict, dict):
52
+ task_id = verdict.get("task_id")
53
+ raw = verdict.get("verdict", verdict.get("passed", verdict.get("result")))
54
+ elif hasattr(verdict, "task_id"):
55
+ task_id = getattr(verdict, "task_id", None)
56
+ for attr in ("verdict", "passed", "result"):
57
+ if hasattr(verdict, attr):
58
+ raw = getattr(verdict, attr)
59
+ break
60
+ if task_id is None:
61
+ if task_list is None:
62
+ raise ValueError(
63
+ "verdicts carry no task_id and no task_ids were provided"
64
+ )
65
+ task_id = task_list[i]
66
+ attempt = counters.get(task_id, 0)
67
+ counters[task_id] = attempt + 1
68
+ runs.append(
69
+ Run(
70
+ task_id=task_id,
71
+ attempt=attempt,
72
+ success=_verdict_to_success(raw),
73
+ version=version,
74
+ metadata={"source": "nitelos"},
75
+ )
76
+ )
77
+ return runs