reliagate 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reliagate/__init__.py +103 -0
- reliagate/__main__.py +4 -0
- reliagate/adapters/__init__.py +5 -0
- reliagate/adapters/jsonl.py +84 -0
- reliagate/adapters/nitelos.py +77 -0
- reliagate/adapters/runner.py +238 -0
- reliagate/cli.py +179 -0
- reliagate/compare.py +301 -0
- reliagate/gate.py +163 -0
- reliagate/metrics.py +290 -0
- reliagate/model.py +131 -0
- reliagate/planning.py +259 -0
- reliagate/stats.py +391 -0
- reliagate-0.2.0.dist-info/METADATA +138 -0
- reliagate-0.2.0.dist-info/RECORD +19 -0
- reliagate-0.2.0.dist-info/WHEEL +5 -0
- reliagate-0.2.0.dist-info/entry_points.txt +2 -0
- reliagate-0.2.0.dist-info/licenses/LICENSE +21 -0
- reliagate-0.2.0.dist-info/top_level.txt +1 -0
reliagate/__init__.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""reliagate -- reliability metrics and CI gates for AI agents."""
|
|
2
|
+
|
|
3
|
+
from .model import (
|
|
4
|
+
EvalRun,
|
|
5
|
+
Run,
|
|
6
|
+
abstention_report,
|
|
7
|
+
consistency_class,
|
|
8
|
+
consistency_classes,
|
|
9
|
+
group_by_task,
|
|
10
|
+
task_counts,
|
|
11
|
+
)
|
|
12
|
+
from .metrics import (
|
|
13
|
+
Estimate,
|
|
14
|
+
bootstrap_pass_pow_k,
|
|
15
|
+
pass_at_k,
|
|
16
|
+
pass_k,
|
|
17
|
+
pass_pow_k,
|
|
18
|
+
per_task,
|
|
19
|
+
consistency,
|
|
20
|
+
wilson_ci,
|
|
21
|
+
wilson_indicator,
|
|
22
|
+
)
|
|
23
|
+
from .compare import (
|
|
24
|
+
ComparisonResult,
|
|
25
|
+
compare,
|
|
26
|
+
mcnemar_exact,
|
|
27
|
+
mcnemar_midp,
|
|
28
|
+
)
|
|
29
|
+
from .stats import (
|
|
30
|
+
WilcoxonResult,
|
|
31
|
+
average_ranks,
|
|
32
|
+
benjamini_hochberg_adjust,
|
|
33
|
+
correct_pvalues,
|
|
34
|
+
hierarchical_bootstrap,
|
|
35
|
+
holm_adjust,
|
|
36
|
+
newcombe_paired,
|
|
37
|
+
wilcoxon_signed_rank,
|
|
38
|
+
)
|
|
39
|
+
from .planning import (
|
|
40
|
+
PlanResult,
|
|
41
|
+
SprtResult,
|
|
42
|
+
consistency_settled,
|
|
43
|
+
plan,
|
|
44
|
+
required_runs,
|
|
45
|
+
sprt_stop,
|
|
46
|
+
)
|
|
47
|
+
from .gate import (
|
|
48
|
+
EXIT_ERROR,
|
|
49
|
+
EXIT_INCONCLUSIVE,
|
|
50
|
+
EXIT_PASS,
|
|
51
|
+
EXIT_REGRESSION,
|
|
52
|
+
GateReport,
|
|
53
|
+
gate,
|
|
54
|
+
)
|
|
55
|
+
from .adapters.jsonl import JSONLError, read_jsonl, write_jsonl
|
|
56
|
+
|
|
57
|
+
__version__ = "0.2.0"
|
|
58
|
+
|
|
59
|
+
__all__ = [
|
|
60
|
+
"EvalRun",
|
|
61
|
+
"Run",
|
|
62
|
+
"abstention_report",
|
|
63
|
+
"consistency_class",
|
|
64
|
+
"consistency_classes",
|
|
65
|
+
"group_by_task",
|
|
66
|
+
"task_counts",
|
|
67
|
+
"Estimate",
|
|
68
|
+
"bootstrap_pass_pow_k",
|
|
69
|
+
"pass_at_k",
|
|
70
|
+
"pass_k",
|
|
71
|
+
"pass_pow_k",
|
|
72
|
+
"per_task",
|
|
73
|
+
"consistency",
|
|
74
|
+
"wilson_ci",
|
|
75
|
+
"wilson_indicator",
|
|
76
|
+
"ComparisonResult",
|
|
77
|
+
"compare",
|
|
78
|
+
"mcnemar_exact",
|
|
79
|
+
"mcnemar_midp",
|
|
80
|
+
"WilcoxonResult",
|
|
81
|
+
"average_ranks",
|
|
82
|
+
"benjamini_hochberg_adjust",
|
|
83
|
+
"correct_pvalues",
|
|
84
|
+
"hierarchical_bootstrap",
|
|
85
|
+
"holm_adjust",
|
|
86
|
+
"newcombe_paired",
|
|
87
|
+
"wilcoxon_signed_rank",
|
|
88
|
+
"PlanResult",
|
|
89
|
+
"SprtResult",
|
|
90
|
+
"consistency_settled",
|
|
91
|
+
"plan",
|
|
92
|
+
"required_runs",
|
|
93
|
+
"sprt_stop",
|
|
94
|
+
"EXIT_ERROR",
|
|
95
|
+
"EXIT_INCONCLUSIVE",
|
|
96
|
+
"EXIT_PASS",
|
|
97
|
+
"EXIT_REGRESSION",
|
|
98
|
+
"GateReport",
|
|
99
|
+
"gate",
|
|
100
|
+
"JSONLError",
|
|
101
|
+
"read_jsonl",
|
|
102
|
+
"write_jsonl",
|
|
103
|
+
]
|
reliagate/__main__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""JSONL in/out adapter (stdlib only -- always available).
|
|
2
|
+
|
|
3
|
+
Line schema (one trial per line):
|
|
4
|
+
{"task_id": "checkout", "attempt": 0, "success": true,
|
|
5
|
+
"version": "abc123", "metadata": {"latency_ms": 4311}}
|
|
6
|
+
``success`` is true, false or null (null = abstained/infra error).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..model import EvalRun, Run
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class JSONLError(ValueError):
|
|
18
|
+
"""Raised on malformed JSONL -- the gate maps this to exit 3."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def write_jsonl(run: EvalRun, path: str) -> int:
|
|
22
|
+
"""Write an EvalRun to path; returns the number of lines."""
|
|
23
|
+
count = 0
|
|
24
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
25
|
+
for r in run.runs:
|
|
26
|
+
fh.write(json.dumps(r.to_dict() if hasattr(r, "to_dict") else {
|
|
27
|
+
"task_id": r.task_id,
|
|
28
|
+
"attempt": r.attempt,
|
|
29
|
+
"success": r.success,
|
|
30
|
+
"version": r.version,
|
|
31
|
+
"metadata": r.metadata,
|
|
32
|
+
}) + "\n")
|
|
33
|
+
count += 1
|
|
34
|
+
return count
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_jsonl(path: str, name: str | None = None) -> EvalRun:
|
|
38
|
+
"""Read a JSONL file of trials into an EvalRun.
|
|
39
|
+
|
|
40
|
+
Raises JSONLError (a ValueError) on unreadable files or malformed
|
|
41
|
+
lines -- callers must surface this as an infra error, never as a
|
|
42
|
+
comparison result.
|
|
43
|
+
"""
|
|
44
|
+
runs: list[Run] = []
|
|
45
|
+
try:
|
|
46
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
47
|
+
lines = fh.readlines()
|
|
48
|
+
except OSError as exc:
|
|
49
|
+
raise JSONLError(f"cannot read {path!r}: {exc}") from exc
|
|
50
|
+
|
|
51
|
+
for lineno, raw in enumerate(lines, start=1):
|
|
52
|
+
line = raw.strip()
|
|
53
|
+
if not line:
|
|
54
|
+
continue
|
|
55
|
+
try:
|
|
56
|
+
obj: Any = json.loads(line)
|
|
57
|
+
except json.JSONDecodeError as exc:
|
|
58
|
+
raise JSONLError(f"{path}:{lineno}: not valid JSON: {exc}") from exc
|
|
59
|
+
if not isinstance(obj, dict):
|
|
60
|
+
raise JSONLError(f"{path}:{lineno}: expected a JSON object")
|
|
61
|
+
missing = [k for k in ("task_id", "attempt", "success") if k not in obj]
|
|
62
|
+
if missing:
|
|
63
|
+
raise JSONLError(
|
|
64
|
+
f"{path}:{lineno}: missing required keys {missing}"
|
|
65
|
+
)
|
|
66
|
+
metadata = obj.get("metadata", {})
|
|
67
|
+
if not isinstance(metadata, dict):
|
|
68
|
+
raise JSONLError(f"{path}:{lineno}: 'metadata' must be an object")
|
|
69
|
+
try:
|
|
70
|
+
runs.append(
|
|
71
|
+
Run(
|
|
72
|
+
task_id=obj["task_id"],
|
|
73
|
+
attempt=obj["attempt"],
|
|
74
|
+
success=obj["success"],
|
|
75
|
+
version=obj.get("version"),
|
|
76
|
+
metadata=metadata,
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
except (TypeError, ValueError) as exc:
|
|
80
|
+
raise JSONLError(f"{path}:{lineno}: {exc}") from exc
|
|
81
|
+
|
|
82
|
+
if not runs:
|
|
83
|
+
raise JSONLError(f"{path}: contains no trials")
|
|
84
|
+
return EvalRun(name=name or path, runs=runs)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""nitelos verdicts adapter (optional, lazy import).
|
|
2
|
+
|
|
3
|
+
Maps nitelos verdict objects to reliagate Run trials. nitelos is NOT a
|
|
4
|
+
dependency of the core; import this module only when nitelos is installed.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, Iterable
|
|
10
|
+
|
|
11
|
+
from ..model import Run
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _verdict_to_success(verdict: Any) -> bool | None:
|
|
15
|
+
"""Map a nitelos verdict to True/False/None.
|
|
16
|
+
|
|
17
|
+
Known mappings (documented, best-effort -- nitelos is not pinned here):
|
|
18
|
+
* truthy string 'pass'/'passed'/'success'/'true' -> True
|
|
19
|
+
* 'fail'/'failed'/'false' -> False
|
|
20
|
+
* 'error'/'abstain'/'skipped'/None -> None (abstained, never a failure)
|
|
21
|
+
"""
|
|
22
|
+
if verdict is None:
|
|
23
|
+
return None
|
|
24
|
+
if isinstance(verdict, bool):
|
|
25
|
+
return verdict
|
|
26
|
+
text = str(verdict).strip().lower()
|
|
27
|
+
if text in ("pass", "passed", "success", "true", "1"):
|
|
28
|
+
return True
|
|
29
|
+
if text in ("fail", "failed", "false", "0"):
|
|
30
|
+
return False
|
|
31
|
+
return None # error/abstain/unknown -> abstained
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def verdicts_to_runs(
|
|
35
|
+
verdicts: Iterable[Any],
|
|
36
|
+
*,
|
|
37
|
+
task_ids: Iterable[str] | None = None,
|
|
38
|
+
version: str | None = None,
|
|
39
|
+
) -> list[Run]:
|
|
40
|
+
"""Convert an iterable of nitelos verdicts to reliagate Runs.
|
|
41
|
+
|
|
42
|
+
``task_ids`` must align with ``verdicts`` when the verdicts do not
|
|
43
|
+
carry their own task identity (attribute ``task_id`` or mapping key).
|
|
44
|
+
"""
|
|
45
|
+
task_list = list(task_ids) if task_ids is not None else None
|
|
46
|
+
runs: list[Run] = []
|
|
47
|
+
counters: dict[str, int] = {}
|
|
48
|
+
for i, verdict in enumerate(verdicts):
|
|
49
|
+
task_id: str | None = None
|
|
50
|
+
raw = verdict
|
|
51
|
+
if isinstance(verdict, dict):
|
|
52
|
+
task_id = verdict.get("task_id")
|
|
53
|
+
raw = verdict.get("verdict", verdict.get("passed", verdict.get("result")))
|
|
54
|
+
elif hasattr(verdict, "task_id"):
|
|
55
|
+
task_id = getattr(verdict, "task_id", None)
|
|
56
|
+
for attr in ("verdict", "passed", "result"):
|
|
57
|
+
if hasattr(verdict, attr):
|
|
58
|
+
raw = getattr(verdict, attr)
|
|
59
|
+
break
|
|
60
|
+
if task_id is None:
|
|
61
|
+
if task_list is None:
|
|
62
|
+
raise ValueError(
|
|
63
|
+
"verdicts carry no task_id and no task_ids were provided"
|
|
64
|
+
)
|
|
65
|
+
task_id = task_list[i]
|
|
66
|
+
attempt = counters.get(task_id, 0)
|
|
67
|
+
counters[task_id] = attempt + 1
|
|
68
|
+
runs.append(
|
|
69
|
+
Run(
|
|
70
|
+
task_id=task_id,
|
|
71
|
+
attempt=attempt,
|
|
72
|
+
success=_verdict_to_success(raw),
|
|
73
|
+
version=version,
|
|
74
|
+
metadata={"source": "nitelos"},
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
return runs
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Generic agent runner with concurrency, a cost budget -- and a live LLM path.
|
|
2
|
+
|
|
3
|
+
The live path uses the owner's Z.ai Coding Plan as THE documented endpoint
|
|
4
|
+
for all live runs of this library:
|
|
5
|
+
|
|
6
|
+
base URL: https://api.z.ai/api/coding/paas/v4 (OpenAI-compatible)
|
|
7
|
+
model: glm-5.3-flash (pinned; no other model is offered)
|
|
8
|
+
key: RELIAGATE_ZAI_KEY environment variable
|
|
9
|
+
|
|
10
|
+
The live path is strictly opt-in behind the env var: nothing in reliagate
|
|
11
|
+
calls a network by default, and all core statistics + tests are offline
|
|
12
|
+
and key-free. On any billing/endpoint error the runner STOPS and raises
|
|
13
|
+
-- it never falls back to another provider or to pay-per-call endpoints.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import concurrent.futures
|
|
19
|
+
import os
|
|
20
|
+
import time
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Callable, Optional, Sequence
|
|
23
|
+
|
|
24
|
+
from ..model import EvalRun, Run
|
|
25
|
+
|
|
26
|
+
ZAI_BASE_URL = "https://api.z.ai/api/coding/paas/v4"
|
|
27
|
+
ZAI_MODEL = "glm-5.3-flash" # pinned by owner; do not add alternatives
|
|
28
|
+
ZAI_KEY_ENV = "RELIAGATE_ZAI_KEY"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
# generic runner
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
AgentFn = Callable[[str], Optional[bool]]
|
|
36
|
+
"""fn(task_id or task payload) -> True/False, or None for abstain/infra error.
|
|
37
|
+
May instead return (success, cost) where cost is a float in budget units."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class RunnerReport:
|
|
42
|
+
run: EvalRun
|
|
43
|
+
cost_spent: float
|
|
44
|
+
attempts_run: int
|
|
45
|
+
attempts_skipped_budget: int
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def run_agent(
|
|
49
|
+
fn: AgentFn,
|
|
50
|
+
tasks: Sequence[str],
|
|
51
|
+
k: int = 1,
|
|
52
|
+
max_workers: int = 4,
|
|
53
|
+
cost_budget: Optional[float] = None,
|
|
54
|
+
estimated_cost_per_call: Optional[float] = None,
|
|
55
|
+
version: Optional[str] = None,
|
|
56
|
+
ordered: bool = True,
|
|
57
|
+
) -> RunnerReport:
|
|
58
|
+
"""Run ``fn`` on every task k times, with a thread pool and cost budget.
|
|
59
|
+
|
|
60
|
+
``cost_budget`` plus ``estimated_cost_per_call``: an attempt is only
|
|
61
|
+
launched when ``cost_spent + estimated_cost_per_call <= budget``.
|
|
62
|
+
Without an estimate, the budget is checked before each batch and a
|
|
63
|
+
batch may overshoot by up to (batch size - 1) observed costs.
|
|
64
|
+
|
|
65
|
+
Attempts not launched because the budget ran out are recorded as
|
|
66
|
+
``success=None`` with metadata ``{"reason": "budget_exhausted"}`` --
|
|
67
|
+
abstained, never counted as failure.
|
|
68
|
+
"""
|
|
69
|
+
if k < 1:
|
|
70
|
+
raise ValueError("k must be >= 1")
|
|
71
|
+
if max_workers < 1:
|
|
72
|
+
raise ValueError("max_workers must be >= 1")
|
|
73
|
+
|
|
74
|
+
def over_budget(spent: float) -> bool:
|
|
75
|
+
if cost_budget is None:
|
|
76
|
+
return False
|
|
77
|
+
if estimated_cost_per_call is not None:
|
|
78
|
+
return spent + estimated_cost_per_call > cost_budget
|
|
79
|
+
return spent >= cost_budget
|
|
80
|
+
|
|
81
|
+
jobs = [(task, attempt) for task in tasks for attempt in range(k)]
|
|
82
|
+
cost_spent = 0.0
|
|
83
|
+
skipped = 0
|
|
84
|
+
results: list[Run] = []
|
|
85
|
+
|
|
86
|
+
def execute(job):
|
|
87
|
+
task, attempt = job
|
|
88
|
+
out = fn(task)
|
|
89
|
+
if isinstance(out, tuple) and len(out) == 2:
|
|
90
|
+
success, cost = out
|
|
91
|
+
else:
|
|
92
|
+
success, cost = out, 0.0
|
|
93
|
+
return task, attempt, success, float(cost)
|
|
94
|
+
|
|
95
|
+
# sequential honouring of the budget; small batches to keep order stable
|
|
96
|
+
pending = list(jobs)
|
|
97
|
+
if not ordered:
|
|
98
|
+
pending.reverse() # executor order is unpredictable anyway
|
|
99
|
+
|
|
100
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
|
|
101
|
+
batch: list = []
|
|
102
|
+
while pending:
|
|
103
|
+
task, attempt = pending.pop(0)
|
|
104
|
+
committed = cost_spent + len(batch) * (
|
|
105
|
+
estimated_cost_per_call or 0.0
|
|
106
|
+
)
|
|
107
|
+
if over_budget(committed):
|
|
108
|
+
skipped += 1 + len(pending)
|
|
109
|
+
results.extend(
|
|
110
|
+
Run(
|
|
111
|
+
task_id=t,
|
|
112
|
+
attempt=a,
|
|
113
|
+
success=None,
|
|
114
|
+
version=version,
|
|
115
|
+
metadata={"reason": "budget_exhausted"},
|
|
116
|
+
)
|
|
117
|
+
for t, a in [ (task, attempt) ] + pending
|
|
118
|
+
)
|
|
119
|
+
break
|
|
120
|
+
batch.append((task, attempt))
|
|
121
|
+
if len(batch) >= max_workers or not pending:
|
|
122
|
+
for future in concurrent.futures.as_completed(
|
|
123
|
+
[pool.submit(execute, j) for j in batch]
|
|
124
|
+
):
|
|
125
|
+
t, a, success, cost = future.result()
|
|
126
|
+
cost_spent += cost
|
|
127
|
+
results.append(
|
|
128
|
+
Run(
|
|
129
|
+
task_id=t,
|
|
130
|
+
attempt=a,
|
|
131
|
+
success=success,
|
|
132
|
+
version=version,
|
|
133
|
+
metadata={"cost": cost},
|
|
134
|
+
)
|
|
135
|
+
)
|
|
136
|
+
batch = []
|
|
137
|
+
|
|
138
|
+
results.sort(key=lambda r: (r.task_id, r.attempt))
|
|
139
|
+
return RunnerReport(
|
|
140
|
+
run=EvalRun(name=version or "run", runs=results),
|
|
141
|
+
cost_spent=cost_spent,
|
|
142
|
+
attempts_run=len(jobs) - skipped,
|
|
143
|
+
attempts_skipped_budget=skipped,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ---------------------------------------------------------------------------
|
|
148
|
+
# live Z.ai endpoint (opt-in behind RELIAGATE_ZAI_KEY)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class ZaiBillingError(RuntimeError):
|
|
152
|
+
"""Billing/endpoint failure of the Z.ai Coding Plan endpoint.
|
|
153
|
+
|
|
154
|
+
Raised instead of falling back: the owner's rule is STOP on billing or
|
|
155
|
+
endpoint errors -- never switch providers, never go pay-per-call.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def make_zai_agent_fn(
|
|
160
|
+
prompt_builder: Callable[[str], str],
|
|
161
|
+
*,
|
|
162
|
+
model: str = ZAI_MODEL,
|
|
163
|
+
base_url: str = ZAI_BASE_URL,
|
|
164
|
+
api_key_env: str = ZAI_KEY_ENV,
|
|
165
|
+
timeout_s: float = 120.0,
|
|
166
|
+
cost_per_call: float = 0.0,
|
|
167
|
+
judge: Callable[[str], bool] | None = None,
|
|
168
|
+
) -> AgentFn:
|
|
169
|
+
"""Build an AgentFn that calls the Z.ai Coding Plan endpoint.
|
|
170
|
+
|
|
171
|
+
Only stdlib (urllib) is used. The call happens ONLY if the key env
|
|
172
|
+
var is set, and this factory is only called from live scripts -- the
|
|
173
|
+
library core never touches it.
|
|
174
|
+
|
|
175
|
+
``judge`` maps the model's completion text to a boolean verdict
|
|
176
|
+
(task-specific); omit it to make the agent abstain-friendly by
|
|
177
|
+
returning None on unparseable output.
|
|
178
|
+
"""
|
|
179
|
+
import json as _json
|
|
180
|
+
import urllib.error
|
|
181
|
+
import urllib.request
|
|
182
|
+
|
|
183
|
+
api_key = os.environ.get(api_key_env)
|
|
184
|
+
if not api_key:
|
|
185
|
+
raise RuntimeError(
|
|
186
|
+
f"live runs are opt-in: set {api_key_env} to use the Z.ai Coding "
|
|
187
|
+
f"Plan endpoint ({base_url}, model {model})"
|
|
188
|
+
)
|
|
189
|
+
if model != ZAI_MODEL:
|
|
190
|
+
# owner rule: glm-5.3-flash is the only supported model
|
|
191
|
+
raise ValueError(
|
|
192
|
+
f"only the pinned model {ZAI_MODEL!r} may be used (got {model!r})"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def call(task: str) -> Optional[bool]:
|
|
196
|
+
url = base_url.rstrip("/") + "/chat/completions"
|
|
197
|
+
payload = _json.dumps(
|
|
198
|
+
{
|
|
199
|
+
"model": model,
|
|
200
|
+
"messages": [{"role": "user", "content": prompt_builder(task)}],
|
|
201
|
+
}
|
|
202
|
+
).encode("utf-8")
|
|
203
|
+
req = urllib.request.Request(
|
|
204
|
+
url,
|
|
205
|
+
data=payload,
|
|
206
|
+
headers={
|
|
207
|
+
"Content-Type": "application/json",
|
|
208
|
+
"Authorization": f"Bearer {api_key}",
|
|
209
|
+
},
|
|
210
|
+
)
|
|
211
|
+
started = time.monotonic()
|
|
212
|
+
try:
|
|
213
|
+
with urllib.request.urlopen(req, timeout=timeout_s) as resp:
|
|
214
|
+
body = _json.loads(resp.read().decode("utf-8"))
|
|
215
|
+
except urllib.error.HTTPError as exc:
|
|
216
|
+
# 401/402/429 = auth/billing/rate-limit -> STOP, no fallback
|
|
217
|
+
raise ZaiBillingError(
|
|
218
|
+
f"Z.ai endpoint error (HTTP {exc.code}) after "
|
|
219
|
+
f"{time.monotonic() - started:.1f}s: {exc.reason}; "
|
|
220
|
+
"stopping -- no provider fallback per owner policy"
|
|
221
|
+
) from exc
|
|
222
|
+
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
223
|
+
raise ZaiBillingError(
|
|
224
|
+
f"Z.ai endpoint unreachable: {exc!r}; "
|
|
225
|
+
"stopping -- no provider fallback per owner policy"
|
|
226
|
+
) from exc
|
|
227
|
+
try:
|
|
228
|
+
text = body["choices"][0]["message"]["content"]
|
|
229
|
+
except (KeyError, IndexError, TypeError):
|
|
230
|
+
return None
|
|
231
|
+
if judge is None:
|
|
232
|
+
return None
|
|
233
|
+
try:
|
|
234
|
+
return bool(judge(text))
|
|
235
|
+
except Exception:
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
return call
|
reliagate/cli.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""reliagate command-line interface.
|
|
2
|
+
|
|
3
|
+
Subcommands:
|
|
4
|
+
gate CI gate over two JSONL trial files.
|
|
5
|
+
compare human-readable comparison report, no exit code semantics.
|
|
6
|
+
|
|
7
|
+
Exit codes for `gate` (see reliagate.gate):
|
|
8
|
+
0 pass -- 2 significant regression -- 3 infra/usage error -- 4 inconclusive.
|
|
9
|
+
Argument/usage errors also exit 3, never argparse's default 2, so "2"
|
|
10
|
+
always means exactly one thing: statistical regression.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
from .adapters.jsonl import JSONLError, read_jsonl
|
|
20
|
+
from .compare import compare
|
|
21
|
+
from .gate import EXIT_ERROR, EXIT_PASS, EXIT_REGRESSION, gate
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _ArgErrorParser(argparse.ArgumentParser):
|
|
25
|
+
"""argparse that exits 3 (infra/usage) instead of the default 2."""
|
|
26
|
+
|
|
27
|
+
def error(self, message: str): # type: ignore[override]
|
|
28
|
+
self.print_usage(sys.stderr)
|
|
29
|
+
print(f"{self.prog}: error: {message}", file=sys.stderr)
|
|
30
|
+
raise SystemExit(EXIT_ERROR)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _build_parser() -> _ArgErrorParser:
|
|
34
|
+
parser = _ArgErrorParser(
|
|
35
|
+
prog="reliagate",
|
|
36
|
+
description="reliability metrics and CI gates for AI agents",
|
|
37
|
+
)
|
|
38
|
+
parser.add_argument(
|
|
39
|
+
"--version", action="store_true", help="print version and exit"
|
|
40
|
+
)
|
|
41
|
+
sub = parser.add_subparsers(dest="command")
|
|
42
|
+
|
|
43
|
+
g = sub.add_parser(
|
|
44
|
+
"gate",
|
|
45
|
+
help="CI gate: exit 0 pass / 2 significant regression / 3 infra error / 4 inconclusive",
|
|
46
|
+
)
|
|
47
|
+
g.add_argument("--base", required=True, help="base version trials (JSONL)")
|
|
48
|
+
g.add_argument("--new", required=True, help="candidate trials (JSONL)")
|
|
49
|
+
g.add_argument(
|
|
50
|
+
"--max-regression",
|
|
51
|
+
type=float,
|
|
52
|
+
default=0.05,
|
|
53
|
+
help="max tolerated drop in consistently-solved task fraction "
|
|
54
|
+
"(default 0.05)",
|
|
55
|
+
)
|
|
56
|
+
g.add_argument(
|
|
57
|
+
"--alpha", type=float, default=0.05, help="significance level (default 0.05)"
|
|
58
|
+
)
|
|
59
|
+
g.add_argument(
|
|
60
|
+
"--max-abstention",
|
|
61
|
+
type=float,
|
|
62
|
+
default=0.05,
|
|
63
|
+
help="max share of abstained/infra-errored trials in either version "
|
|
64
|
+
"before the gate refuses to judge (exit 3; default 0.05)",
|
|
65
|
+
)
|
|
66
|
+
g.add_argument(
|
|
67
|
+
"--json", action="store_true", help="print the full report as JSON"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
c = sub.add_parser("compare", help="human-readable paired comparison")
|
|
71
|
+
c.add_argument("--base", required=True)
|
|
72
|
+
c.add_argument("--new", required=True)
|
|
73
|
+
return parser
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _load(path: str):
|
|
77
|
+
try:
|
|
78
|
+
return read_jsonl(path)
|
|
79
|
+
except JSONLError as exc:
|
|
80
|
+
print(f"reliagate: infra error: {exc}", file=sys.stderr)
|
|
81
|
+
raise SystemExit(EXIT_ERROR)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def main(argv: list[str] | None = None) -> int:
|
|
85
|
+
parser = _build_parser()
|
|
86
|
+
args = parser.parse_args(argv)
|
|
87
|
+
if args.version:
|
|
88
|
+
from . import __version__
|
|
89
|
+
|
|
90
|
+
print(f"reliagate {__version__}")
|
|
91
|
+
return EXIT_PASS
|
|
92
|
+
if args.command == "gate":
|
|
93
|
+
base = _load(args.base)
|
|
94
|
+
new = _load(args.new)
|
|
95
|
+
report = gate(
|
|
96
|
+
base, new, max_regression=args.max_regression, alpha=args.alpha,
|
|
97
|
+
max_abstention=args.max_abstention,
|
|
98
|
+
)
|
|
99
|
+
if args.json:
|
|
100
|
+
print(json.dumps(report.to_dict(), indent=2))
|
|
101
|
+
else:
|
|
102
|
+
cmp = report.comparison
|
|
103
|
+
print(
|
|
104
|
+
f"gate decision: {report.decision.upper()} "
|
|
105
|
+
f"(exit {report.exit_code})"
|
|
106
|
+
)
|
|
107
|
+
if cmp is not None:
|
|
108
|
+
print(
|
|
109
|
+
f" consistently solved: base {cmp.base_consistent}/"
|
|
110
|
+
f"{cmp.n_tasks} -> new {cmp.new_consistent}/{cmp.n_tasks}"
|
|
111
|
+
)
|
|
112
|
+
print(
|
|
113
|
+
f" gained (b)={cmp.b} lost (c)={cmp.c} b-c={cmp.b_minus_c} "
|
|
114
|
+
f"test={cmp.test_used} p={cmp.p_value:.4g}"
|
|
115
|
+
)
|
|
116
|
+
for change in cmp.class_changes:
|
|
117
|
+
print(
|
|
118
|
+
f" class change: {change['task_id']}: "
|
|
119
|
+
f"{change['base']} -> {change['new']}"
|
|
120
|
+
)
|
|
121
|
+
print(f" reason: {report.reason}")
|
|
122
|
+
ab = report.abstention
|
|
123
|
+
if ab: # empty when the gate rejected its arguments before loading runs
|
|
124
|
+
print(
|
|
125
|
+
f" abstentions: base {ab['base']['trials_abstained']}/"
|
|
126
|
+
f"{ab['base']['trials_total']} trials, "
|
|
127
|
+
f"new {ab['new']['trials_abstained']}/{ab['new']['trials_total']} "
|
|
128
|
+
f"trials (counted in n, never counted as failure)"
|
|
129
|
+
)
|
|
130
|
+
raise SystemExit(report.exit_code)
|
|
131
|
+
if args.command == "compare":
|
|
132
|
+
base = _load(args.base)
|
|
133
|
+
new = _load(args.new)
|
|
134
|
+
cmp = compare(base, new)
|
|
135
|
+
print(f"tasks: {cmp.n_tasks}")
|
|
136
|
+
print(f"b (gained consistent)={cmp.b} c (lost)={cmp.c} b-c={cmp.b_minus_c}")
|
|
137
|
+
print(f"test={cmp.test_used} p={cmp.p_value:.4g}")
|
|
138
|
+
if cmp.newcombe:
|
|
139
|
+
lo, hi = cmp.newcombe["ci"]
|
|
140
|
+
print(
|
|
141
|
+
f"newcombe paired 95% CI on difference: "
|
|
142
|
+
f"[{lo:+.4f}, {hi:+.4f}] (point {cmp.newcombe['difference']:+.4f})"
|
|
143
|
+
)
|
|
144
|
+
if cmp.wilcoxon and cmp.wilcoxon["method"] != "degenerate":
|
|
145
|
+
print(
|
|
146
|
+
f"wilcoxon signed-rank ({cmp.wilcoxon['method']}, "
|
|
147
|
+
f"n={cmp.wilcoxon['n_nonzero']}): p={cmp.wilcoxon['p_value']:.4g}"
|
|
148
|
+
)
|
|
149
|
+
elif cmp.wilcoxon:
|
|
150
|
+
print("wilcoxon signed-rank: all paired differences are zero "
|
|
151
|
+
"(no evidence of any shift; test undefined)")
|
|
152
|
+
for ct in cmp.category_tests:
|
|
153
|
+
adj = ct["adjusted_p_value"]
|
|
154
|
+
adj_s = f" adj_p={adj:.4g}" if adj is not None else ""
|
|
155
|
+
print(
|
|
156
|
+
f"category {ct['category']!r}: b={ct['b']} c={ct['c']} "
|
|
157
|
+
f"p={ct['p_value']:.4g} ({ct['test_used']}){adj_s}"
|
|
158
|
+
)
|
|
159
|
+
if cmp.correction:
|
|
160
|
+
print(
|
|
161
|
+
f"correction: {cmp.correction['method']} over "
|
|
162
|
+
f"{cmp.correction['n_hypotheses']} categories "
|
|
163
|
+
f"({cmp.correction['controls']})"
|
|
164
|
+
)
|
|
165
|
+
for change in cmp.class_changes:
|
|
166
|
+
print(
|
|
167
|
+
f"class change: {change['task_id']}: "
|
|
168
|
+
f"{change['base']} -> {change['new']}"
|
|
169
|
+
)
|
|
170
|
+
return EXIT_PASS
|
|
171
|
+
parser.print_help()
|
|
172
|
+
return EXIT_PASS
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
if __name__ == "__main__":
|
|
176
|
+
try:
|
|
177
|
+
main()
|
|
178
|
+
except SystemExit:
|
|
179
|
+
raise
|