reliagate 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reliagate/__init__.py ADDED
@@ -0,0 +1,103 @@
1
+ """reliagate -- reliability metrics and CI gates for AI agents."""
2
+
3
+ from .model import (
4
+ EvalRun,
5
+ Run,
6
+ abstention_report,
7
+ consistency_class,
8
+ consistency_classes,
9
+ group_by_task,
10
+ task_counts,
11
+ )
12
+ from .metrics import (
13
+ Estimate,
14
+ bootstrap_pass_pow_k,
15
+ pass_at_k,
16
+ pass_k,
17
+ pass_pow_k,
18
+ per_task,
19
+ consistency,
20
+ wilson_ci,
21
+ wilson_indicator,
22
+ )
23
+ from .compare import (
24
+ ComparisonResult,
25
+ compare,
26
+ mcnemar_exact,
27
+ mcnemar_midp,
28
+ )
29
+ from .stats import (
30
+ WilcoxonResult,
31
+ average_ranks,
32
+ benjamini_hochberg_adjust,
33
+ correct_pvalues,
34
+ hierarchical_bootstrap,
35
+ holm_adjust,
36
+ newcombe_paired,
37
+ wilcoxon_signed_rank,
38
+ )
39
+ from .planning import (
40
+ PlanResult,
41
+ SprtResult,
42
+ consistency_settled,
43
+ plan,
44
+ required_runs,
45
+ sprt_stop,
46
+ )
47
+ from .gate import (
48
+ EXIT_ERROR,
49
+ EXIT_INCONCLUSIVE,
50
+ EXIT_PASS,
51
+ EXIT_REGRESSION,
52
+ GateReport,
53
+ gate,
54
+ )
55
+ from .adapters.jsonl import JSONLError, read_jsonl, write_jsonl
56
+
57
+ __version__ = "0.2.0"
58
+
59
+ __all__ = [
60
+ "EvalRun",
61
+ "Run",
62
+ "abstention_report",
63
+ "consistency_class",
64
+ "consistency_classes",
65
+ "group_by_task",
66
+ "task_counts",
67
+ "Estimate",
68
+ "bootstrap_pass_pow_k",
69
+ "pass_at_k",
70
+ "pass_k",
71
+ "pass_pow_k",
72
+ "per_task",
73
+ "consistency",
74
+ "wilson_ci",
75
+ "wilson_indicator",
76
+ "ComparisonResult",
77
+ "compare",
78
+ "mcnemar_exact",
79
+ "mcnemar_midp",
80
+ "WilcoxonResult",
81
+ "average_ranks",
82
+ "benjamini_hochberg_adjust",
83
+ "correct_pvalues",
84
+ "hierarchical_bootstrap",
85
+ "holm_adjust",
86
+ "newcombe_paired",
87
+ "wilcoxon_signed_rank",
88
+ "PlanResult",
89
+ "SprtResult",
90
+ "consistency_settled",
91
+ "plan",
92
+ "required_runs",
93
+ "sprt_stop",
94
+ "EXIT_ERROR",
95
+ "EXIT_INCONCLUSIVE",
96
+ "EXIT_PASS",
97
+ "EXIT_REGRESSION",
98
+ "GateReport",
99
+ "gate",
100
+ "JSONLError",
101
+ "read_jsonl",
102
+ "write_jsonl",
103
+ ]
reliagate/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
@@ -0,0 +1,5 @@
1
+ """Adapters: JSONL (always available), nitelos + Z.ai runner (opt-in).
2
+
3
+ Inspect AI and Langfuse adapters are documented stubs -- see ROADMAP.md.
4
+ Nothing here is imported by the statistics core.
5
+ """
@@ -0,0 +1,84 @@
1
+ """JSONL in/out adapter (stdlib only -- always available).
2
+
3
+ Line schema (one trial per line):
4
+ {"task_id": "checkout", "attempt": 0, "success": true,
5
+ "version": "abc123", "metadata": {"latency_ms": 4311}}
6
+ ``success`` is true, false or null (null = abstained/infra error).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from typing import Any
13
+
14
+ from ..model import EvalRun, Run
15
+
16
+
17
+ class JSONLError(ValueError):
18
+ """Raised on malformed JSONL -- the gate maps this to exit 3."""
19
+
20
+
21
+ def write_jsonl(run: EvalRun, path: str) -> int:
22
+ """Write an EvalRun to path; returns the number of lines."""
23
+ count = 0
24
+ with open(path, "w", encoding="utf-8") as fh:
25
+ for r in run.runs:
26
+ fh.write(json.dumps(r.to_dict() if hasattr(r, "to_dict") else {
27
+ "task_id": r.task_id,
28
+ "attempt": r.attempt,
29
+ "success": r.success,
30
+ "version": r.version,
31
+ "metadata": r.metadata,
32
+ }) + "\n")
33
+ count += 1
34
+ return count
35
+
36
+
37
+ def read_jsonl(path: str, name: str | None = None) -> EvalRun:
38
+ """Read a JSONL file of trials into an EvalRun.
39
+
40
+ Raises JSONLError (a ValueError) on unreadable files or malformed
41
+ lines -- callers must surface this as an infra error, never as a
42
+ comparison result.
43
+ """
44
+ runs: list[Run] = []
45
+ try:
46
+ with open(path, "r", encoding="utf-8") as fh:
47
+ lines = fh.readlines()
48
+ except OSError as exc:
49
+ raise JSONLError(f"cannot read {path!r}: {exc}") from exc
50
+
51
+ for lineno, raw in enumerate(lines, start=1):
52
+ line = raw.strip()
53
+ if not line:
54
+ continue
55
+ try:
56
+ obj: Any = json.loads(line)
57
+ except json.JSONDecodeError as exc:
58
+ raise JSONLError(f"{path}:{lineno}: not valid JSON: {exc}") from exc
59
+ if not isinstance(obj, dict):
60
+ raise JSONLError(f"{path}:{lineno}: expected a JSON object")
61
+ missing = [k for k in ("task_id", "attempt", "success") if k not in obj]
62
+ if missing:
63
+ raise JSONLError(
64
+ f"{path}:{lineno}: missing required keys {missing}"
65
+ )
66
+ metadata = obj.get("metadata", {})
67
+ if not isinstance(metadata, dict):
68
+ raise JSONLError(f"{path}:{lineno}: 'metadata' must be an object")
69
+ try:
70
+ runs.append(
71
+ Run(
72
+ task_id=obj["task_id"],
73
+ attempt=obj["attempt"],
74
+ success=obj["success"],
75
+ version=obj.get("version"),
76
+ metadata=metadata,
77
+ )
78
+ )
79
+ except (TypeError, ValueError) as exc:
80
+ raise JSONLError(f"{path}:{lineno}: {exc}") from exc
81
+
82
+ if not runs:
83
+ raise JSONLError(f"{path}: contains no trials")
84
+ return EvalRun(name=name or path, runs=runs)
@@ -0,0 +1,77 @@
1
+ """nitelos verdicts adapter (optional, lazy import).
2
+
3
+ Maps nitelos verdict objects to reliagate Run trials. nitelos is NOT a
4
+ dependency of the core; import this module only when nitelos is installed.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, Iterable
10
+
11
+ from ..model import Run
12
+
13
+
14
+ def _verdict_to_success(verdict: Any) -> bool | None:
15
+ """Map a nitelos verdict to True/False/None.
16
+
17
+ Known mappings (documented, best-effort -- nitelos is not pinned here):
18
+ * truthy string 'pass'/'passed'/'success'/'true' -> True
19
+ * 'fail'/'failed'/'false' -> False
20
+ * 'error'/'abstain'/'skipped'/None -> None (abstained, never a failure)
21
+ """
22
+ if verdict is None:
23
+ return None
24
+ if isinstance(verdict, bool):
25
+ return verdict
26
+ text = str(verdict).strip().lower()
27
+ if text in ("pass", "passed", "success", "true", "1"):
28
+ return True
29
+ if text in ("fail", "failed", "false", "0"):
30
+ return False
31
+ return None # error/abstain/unknown -> abstained
32
+
33
+
34
+ def verdicts_to_runs(
35
+ verdicts: Iterable[Any],
36
+ *,
37
+ task_ids: Iterable[str] | None = None,
38
+ version: str | None = None,
39
+ ) -> list[Run]:
40
+ """Convert an iterable of nitelos verdicts to reliagate Runs.
41
+
42
+ ``task_ids`` must align with ``verdicts`` when the verdicts do not
43
+ carry their own task identity (attribute ``task_id`` or mapping key).
44
+ """
45
+ task_list = list(task_ids) if task_ids is not None else None
46
+ runs: list[Run] = []
47
+ counters: dict[str, int] = {}
48
+ for i, verdict in enumerate(verdicts):
49
+ task_id: str | None = None
50
+ raw = verdict
51
+ if isinstance(verdict, dict):
52
+ task_id = verdict.get("task_id")
53
+ raw = verdict.get("verdict", verdict.get("passed", verdict.get("result")))
54
+ elif hasattr(verdict, "task_id"):
55
+ task_id = getattr(verdict, "task_id", None)
56
+ for attr in ("verdict", "passed", "result"):
57
+ if hasattr(verdict, attr):
58
+ raw = getattr(verdict, attr)
59
+ break
60
+ if task_id is None:
61
+ if task_list is None:
62
+ raise ValueError(
63
+ "verdicts carry no task_id and no task_ids were provided"
64
+ )
65
+ task_id = task_list[i]
66
+ attempt = counters.get(task_id, 0)
67
+ counters[task_id] = attempt + 1
68
+ runs.append(
69
+ Run(
70
+ task_id=task_id,
71
+ attempt=attempt,
72
+ success=_verdict_to_success(raw),
73
+ version=version,
74
+ metadata={"source": "nitelos"},
75
+ )
76
+ )
77
+ return runs
@@ -0,0 +1,238 @@
1
+ """Generic agent runner with concurrency, a cost budget -- and a live LLM path.
2
+
3
+ The live path uses the owner's Z.ai Coding Plan as THE documented endpoint
4
+ for all live runs of this library:
5
+
6
+ base URL: https://api.z.ai/api/coding/paas/v4 (OpenAI-compatible)
7
+ model: glm-5.3-flash (pinned; no other model is offered)
8
+ key: RELIAGATE_ZAI_KEY environment variable
9
+
10
+ The live path is strictly opt-in behind the env var: nothing in reliagate
11
+ calls a network by default, and all core statistics + tests are offline
12
+ and key-free. On any billing/endpoint error the runner STOPS and raises
13
+ -- it never falls back to another provider or to pay-per-call endpoints.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import concurrent.futures
19
+ import os
20
+ import time
21
+ from dataclasses import dataclass
22
+ from typing import Callable, Optional, Sequence
23
+
24
+ from ..model import EvalRun, Run
25
+
26
+ ZAI_BASE_URL = "https://api.z.ai/api/coding/paas/v4"
27
+ ZAI_MODEL = "glm-5.3-flash" # pinned by owner; do not add alternatives
28
+ ZAI_KEY_ENV = "RELIAGATE_ZAI_KEY"
29
+
30
+
31
+ # ---------------------------------------------------------------------------
32
+ # generic runner
33
+
34
+
35
+ AgentFn = Callable[[str], Optional[bool]]
36
+ """fn(task_id or task payload) -> True/False, or None for abstain/infra error.
37
+ May instead return (success, cost) where cost is a float in budget units."""
38
+
39
+
40
+ @dataclass
41
+ class RunnerReport:
42
+ run: EvalRun
43
+ cost_spent: float
44
+ attempts_run: int
45
+ attempts_skipped_budget: int
46
+
47
+
48
+ def run_agent(
49
+ fn: AgentFn,
50
+ tasks: Sequence[str],
51
+ k: int = 1,
52
+ max_workers: int = 4,
53
+ cost_budget: Optional[float] = None,
54
+ estimated_cost_per_call: Optional[float] = None,
55
+ version: Optional[str] = None,
56
+ ordered: bool = True,
57
+ ) -> RunnerReport:
58
+ """Run ``fn`` on every task k times, with a thread pool and cost budget.
59
+
60
+ ``cost_budget`` plus ``estimated_cost_per_call``: an attempt is only
61
+ launched when ``cost_spent + estimated_cost_per_call <= budget``.
62
+ Without an estimate, the budget is checked before each batch and a
63
+ batch may overshoot by up to (batch size - 1) observed costs.
64
+
65
+ Attempts not launched because the budget ran out are recorded as
66
+ ``success=None`` with metadata ``{"reason": "budget_exhausted"}`` --
67
+ abstained, never counted as failure.
68
+ """
69
+ if k < 1:
70
+ raise ValueError("k must be >= 1")
71
+ if max_workers < 1:
72
+ raise ValueError("max_workers must be >= 1")
73
+
74
+ def over_budget(spent: float) -> bool:
75
+ if cost_budget is None:
76
+ return False
77
+ if estimated_cost_per_call is not None:
78
+ return spent + estimated_cost_per_call > cost_budget
79
+ return spent >= cost_budget
80
+
81
+ jobs = [(task, attempt) for task in tasks for attempt in range(k)]
82
+ cost_spent = 0.0
83
+ skipped = 0
84
+ results: list[Run] = []
85
+
86
+ def execute(job):
87
+ task, attempt = job
88
+ out = fn(task)
89
+ if isinstance(out, tuple) and len(out) == 2:
90
+ success, cost = out
91
+ else:
92
+ success, cost = out, 0.0
93
+ return task, attempt, success, float(cost)
94
+
95
+ # sequential honouring of the budget; small batches to keep order stable
96
+ pending = list(jobs)
97
+ if not ordered:
98
+ pending.reverse() # executor order is unpredictable anyway
99
+
100
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
101
+ batch: list = []
102
+ while pending:
103
+ task, attempt = pending.pop(0)
104
+ committed = cost_spent + len(batch) * (
105
+ estimated_cost_per_call or 0.0
106
+ )
107
+ if over_budget(committed):
108
+ skipped += 1 + len(pending)
109
+ results.extend(
110
+ Run(
111
+ task_id=t,
112
+ attempt=a,
113
+ success=None,
114
+ version=version,
115
+ metadata={"reason": "budget_exhausted"},
116
+ )
117
+ for t, a in [ (task, attempt) ] + pending
118
+ )
119
+ break
120
+ batch.append((task, attempt))
121
+ if len(batch) >= max_workers or not pending:
122
+ for future in concurrent.futures.as_completed(
123
+ [pool.submit(execute, j) for j in batch]
124
+ ):
125
+ t, a, success, cost = future.result()
126
+ cost_spent += cost
127
+ results.append(
128
+ Run(
129
+ task_id=t,
130
+ attempt=a,
131
+ success=success,
132
+ version=version,
133
+ metadata={"cost": cost},
134
+ )
135
+ )
136
+ batch = []
137
+
138
+ results.sort(key=lambda r: (r.task_id, r.attempt))
139
+ return RunnerReport(
140
+ run=EvalRun(name=version or "run", runs=results),
141
+ cost_spent=cost_spent,
142
+ attempts_run=len(jobs) - skipped,
143
+ attempts_skipped_budget=skipped,
144
+ )
145
+
146
+
147
+ # ---------------------------------------------------------------------------
148
+ # live Z.ai endpoint (opt-in behind RELIAGATE_ZAI_KEY)
149
+
150
+
151
+ class ZaiBillingError(RuntimeError):
152
+ """Billing/endpoint failure of the Z.ai Coding Plan endpoint.
153
+
154
+ Raised instead of falling back: the owner's rule is STOP on billing or
155
+ endpoint errors -- never switch providers, never go pay-per-call.
156
+ """
157
+
158
+
159
+ def make_zai_agent_fn(
160
+ prompt_builder: Callable[[str], str],
161
+ *,
162
+ model: str = ZAI_MODEL,
163
+ base_url: str = ZAI_BASE_URL,
164
+ api_key_env: str = ZAI_KEY_ENV,
165
+ timeout_s: float = 120.0,
166
+ cost_per_call: float = 0.0,
167
+ judge: Callable[[str], bool] | None = None,
168
+ ) -> AgentFn:
169
+ """Build an AgentFn that calls the Z.ai Coding Plan endpoint.
170
+
171
+ Only stdlib (urllib) is used. The call happens ONLY if the key env
172
+ var is set, and this factory is only called from live scripts -- the
173
+ library core never touches it.
174
+
175
+ ``judge`` maps the model's completion text to a boolean verdict
176
+ (task-specific); omit it to make the agent abstain-friendly by
177
+ returning None on unparseable output.
178
+ """
179
+ import json as _json
180
+ import urllib.error
181
+ import urllib.request
182
+
183
+ api_key = os.environ.get(api_key_env)
184
+ if not api_key:
185
+ raise RuntimeError(
186
+ f"live runs are opt-in: set {api_key_env} to use the Z.ai Coding "
187
+ f"Plan endpoint ({base_url}, model {model})"
188
+ )
189
+ if model != ZAI_MODEL:
190
+ # owner rule: glm-5.3-flash is the only supported model
191
+ raise ValueError(
192
+ f"only the pinned model {ZAI_MODEL!r} may be used (got {model!r})"
193
+ )
194
+
195
+ def call(task: str) -> Optional[bool]:
196
+ url = base_url.rstrip("/") + "/chat/completions"
197
+ payload = _json.dumps(
198
+ {
199
+ "model": model,
200
+ "messages": [{"role": "user", "content": prompt_builder(task)}],
201
+ }
202
+ ).encode("utf-8")
203
+ req = urllib.request.Request(
204
+ url,
205
+ data=payload,
206
+ headers={
207
+ "Content-Type": "application/json",
208
+ "Authorization": f"Bearer {api_key}",
209
+ },
210
+ )
211
+ started = time.monotonic()
212
+ try:
213
+ with urllib.request.urlopen(req, timeout=timeout_s) as resp:
214
+ body = _json.loads(resp.read().decode("utf-8"))
215
+ except urllib.error.HTTPError as exc:
216
+ # 401/402/429 = auth/billing/rate-limit -> STOP, no fallback
217
+ raise ZaiBillingError(
218
+ f"Z.ai endpoint error (HTTP {exc.code}) after "
219
+ f"{time.monotonic() - started:.1f}s: {exc.reason}; "
220
+ "stopping -- no provider fallback per owner policy"
221
+ ) from exc
222
+ except (urllib.error.URLError, TimeoutError, OSError) as exc:
223
+ raise ZaiBillingError(
224
+ f"Z.ai endpoint unreachable: {exc!r}; "
225
+ "stopping -- no provider fallback per owner policy"
226
+ ) from exc
227
+ try:
228
+ text = body["choices"][0]["message"]["content"]
229
+ except (KeyError, IndexError, TypeError):
230
+ return None
231
+ if judge is None:
232
+ return None
233
+ try:
234
+ return bool(judge(text))
235
+ except Exception:
236
+ return None
237
+
238
+ return call
reliagate/cli.py ADDED
@@ -0,0 +1,179 @@
1
+ """reliagate command-line interface.
2
+
3
+ Subcommands:
4
+ gate CI gate over two JSONL trial files.
5
+ compare human-readable comparison report, no exit code semantics.
6
+
7
+ Exit codes for `gate` (see reliagate.gate):
8
+ 0 pass -- 2 significant regression -- 3 infra/usage error -- 4 inconclusive.
9
+ Argument/usage errors also exit 3, never argparse's default 2, so "2"
10
+ always means exactly one thing: statistical regression.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import sys
18
+
19
+ from .adapters.jsonl import JSONLError, read_jsonl
20
+ from .compare import compare
21
+ from .gate import EXIT_ERROR, EXIT_PASS, EXIT_REGRESSION, gate
22
+
23
+
24
+ class _ArgErrorParser(argparse.ArgumentParser):
25
+ """argparse that exits 3 (infra/usage) instead of the default 2."""
26
+
27
+ def error(self, message: str): # type: ignore[override]
28
+ self.print_usage(sys.stderr)
29
+ print(f"{self.prog}: error: {message}", file=sys.stderr)
30
+ raise SystemExit(EXIT_ERROR)
31
+
32
+
33
+ def _build_parser() -> _ArgErrorParser:
34
+ parser = _ArgErrorParser(
35
+ prog="reliagate",
36
+ description="reliability metrics and CI gates for AI agents",
37
+ )
38
+ parser.add_argument(
39
+ "--version", action="store_true", help="print version and exit"
40
+ )
41
+ sub = parser.add_subparsers(dest="command")
42
+
43
+ g = sub.add_parser(
44
+ "gate",
45
+ help="CI gate: exit 0 pass / 2 significant regression / 3 infra error / 4 inconclusive",
46
+ )
47
+ g.add_argument("--base", required=True, help="base version trials (JSONL)")
48
+ g.add_argument("--new", required=True, help="candidate trials (JSONL)")
49
+ g.add_argument(
50
+ "--max-regression",
51
+ type=float,
52
+ default=0.05,
53
+ help="max tolerated drop in consistently-solved task fraction "
54
+ "(default 0.05)",
55
+ )
56
+ g.add_argument(
57
+ "--alpha", type=float, default=0.05, help="significance level (default 0.05)"
58
+ )
59
+ g.add_argument(
60
+ "--max-abstention",
61
+ type=float,
62
+ default=0.05,
63
+ help="max share of abstained/infra-errored trials in either version "
64
+ "before the gate refuses to judge (exit 3; default 0.05)",
65
+ )
66
+ g.add_argument(
67
+ "--json", action="store_true", help="print the full report as JSON"
68
+ )
69
+
70
+ c = sub.add_parser("compare", help="human-readable paired comparison")
71
+ c.add_argument("--base", required=True)
72
+ c.add_argument("--new", required=True)
73
+ return parser
74
+
75
+
76
+ def _load(path: str):
77
+ try:
78
+ return read_jsonl(path)
79
+ except JSONLError as exc:
80
+ print(f"reliagate: infra error: {exc}", file=sys.stderr)
81
+ raise SystemExit(EXIT_ERROR)
82
+
83
+
84
+ def main(argv: list[str] | None = None) -> int:
85
+ parser = _build_parser()
86
+ args = parser.parse_args(argv)
87
+ if args.version:
88
+ from . import __version__
89
+
90
+ print(f"reliagate {__version__}")
91
+ return EXIT_PASS
92
+ if args.command == "gate":
93
+ base = _load(args.base)
94
+ new = _load(args.new)
95
+ report = gate(
96
+ base, new, max_regression=args.max_regression, alpha=args.alpha,
97
+ max_abstention=args.max_abstention,
98
+ )
99
+ if args.json:
100
+ print(json.dumps(report.to_dict(), indent=2))
101
+ else:
102
+ cmp = report.comparison
103
+ print(
104
+ f"gate decision: {report.decision.upper()} "
105
+ f"(exit {report.exit_code})"
106
+ )
107
+ if cmp is not None:
108
+ print(
109
+ f" consistently solved: base {cmp.base_consistent}/"
110
+ f"{cmp.n_tasks} -> new {cmp.new_consistent}/{cmp.n_tasks}"
111
+ )
112
+ print(
113
+ f" gained (b)={cmp.b} lost (c)={cmp.c} b-c={cmp.b_minus_c} "
114
+ f"test={cmp.test_used} p={cmp.p_value:.4g}"
115
+ )
116
+ for change in cmp.class_changes:
117
+ print(
118
+ f" class change: {change['task_id']}: "
119
+ f"{change['base']} -> {change['new']}"
120
+ )
121
+ print(f" reason: {report.reason}")
122
+ ab = report.abstention
123
+ if ab: # empty when the gate rejected its arguments before loading runs
124
+ print(
125
+ f" abstentions: base {ab['base']['trials_abstained']}/"
126
+ f"{ab['base']['trials_total']} trials, "
127
+ f"new {ab['new']['trials_abstained']}/{ab['new']['trials_total']} "
128
+ f"trials (counted in n, never counted as failure)"
129
+ )
130
+ raise SystemExit(report.exit_code)
131
+ if args.command == "compare":
132
+ base = _load(args.base)
133
+ new = _load(args.new)
134
+ cmp = compare(base, new)
135
+ print(f"tasks: {cmp.n_tasks}")
136
+ print(f"b (gained consistent)={cmp.b} c (lost)={cmp.c} b-c={cmp.b_minus_c}")
137
+ print(f"test={cmp.test_used} p={cmp.p_value:.4g}")
138
+ if cmp.newcombe:
139
+ lo, hi = cmp.newcombe["ci"]
140
+ print(
141
+ f"newcombe paired 95% CI on difference: "
142
+ f"[{lo:+.4f}, {hi:+.4f}] (point {cmp.newcombe['difference']:+.4f})"
143
+ )
144
+ if cmp.wilcoxon and cmp.wilcoxon["method"] != "degenerate":
145
+ print(
146
+ f"wilcoxon signed-rank ({cmp.wilcoxon['method']}, "
147
+ f"n={cmp.wilcoxon['n_nonzero']}): p={cmp.wilcoxon['p_value']:.4g}"
148
+ )
149
+ elif cmp.wilcoxon:
150
+ print("wilcoxon signed-rank: all paired differences are zero "
151
+ "(no evidence of any shift; test undefined)")
152
+ for ct in cmp.category_tests:
153
+ adj = ct["adjusted_p_value"]
154
+ adj_s = f" adj_p={adj:.4g}" if adj is not None else ""
155
+ print(
156
+ f"category {ct['category']!r}: b={ct['b']} c={ct['c']} "
157
+ f"p={ct['p_value']:.4g} ({ct['test_used']}){adj_s}"
158
+ )
159
+ if cmp.correction:
160
+ print(
161
+ f"correction: {cmp.correction['method']} over "
162
+ f"{cmp.correction['n_hypotheses']} categories "
163
+ f"({cmp.correction['controls']})"
164
+ )
165
+ for change in cmp.class_changes:
166
+ print(
167
+ f"class change: {change['task_id']}: "
168
+ f"{change['base']} -> {change['new']}"
169
+ )
170
+ return EXIT_PASS
171
+ parser.print_help()
172
+ return EXIT_PASS
173
+
174
+
175
+ if __name__ == "__main__":
176
+ try:
177
+ main()
178
+ except SystemExit:
179
+ raise