jevify 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. jevify/__init__.py +7 -0
  2. jevify/__main__.py +9 -0
  3. jevify/adapters/__init__.py +1 -0
  4. jevify/adapters/embedding/__init__.py +1 -0
  5. jevify/adapters/embedding/adapter.py +203 -0
  6. jevify/adapters/embedding/local.py +82 -0
  7. jevify/adapters/encoder/__init__.py +1 -0
  8. jevify/adapters/encoder/adapter.py +165 -0
  9. jevify/adapters/encoder/local.py +77 -0
  10. jevify/adapters/endpoint/__init__.py +1 -0
  11. jevify/adapters/endpoint/adapter.py +281 -0
  12. jevify/adapters/endpoint/client.py +114 -0
  13. jevify/adapters/endpoint/dialects.py +166 -0
  14. jevify/adapters/endpoint/probe.py +589 -0
  15. jevify/adapters/endpoint/readout.py +244 -0
  16. jevify/adapters/rerank/__init__.py +1 -0
  17. jevify/adapters/rerank/adapter.py +152 -0
  18. jevify/api/__init__.py +1 -0
  19. jevify/api/app.py +228 -0
  20. jevify/api/schema.py +123 -0
  21. jevify/api/translate.py +239 -0
  22. jevify/cli.py +392 -0
  23. jevify/compose.py +82 -0
  24. jevify/domain/__init__.py +1 -0
  25. jevify/domain/distribution.py +103 -0
  26. jevify/domain/engine.py +152 -0
  27. jevify/domain/questions.py +121 -0
  28. jevify/evaluation/__init__.py +1 -0
  29. jevify/evaluation/calibrate.py +186 -0
  30. jevify/evaluation/controls.py +78 -0
  31. jevify/evaluation/metrics.py +91 -0
  32. jevify/evaluation/scorecard.py +385 -0
  33. jevify/evaluation/suites.py +166 -0
  34. jevify/ports/__init__.py +1 -0
  35. jevify/ports/backend.py +162 -0
  36. jevify/recipes/__init__.py +21 -0
  37. jevify/recipes/probe_case.yaml +72 -0
  38. jevify/recipes/render.py +198 -0
  39. jevify/recipes/schema.py +204 -0
  40. jevify/recipes/store.py +81 -0
  41. jevify-0.1.0.dist-info/METADATA +243 -0
  42. jevify-0.1.0.dist-info/RECORD +45 -0
  43. jevify-0.1.0.dist-info/WHEEL +4 -0
  44. jevify-0.1.0.dist-info/entry_points.txt +2 -0
  45. jevify-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,152 @@
1
+ """The Engine: warm a state, evaluate typed questions, return Answers with semantics.
2
+
3
+ The engine is the only caller of the backend port and the only producer of
4
+ `Answer`. It never renders prompts (the recipe does) and never reads logprobs
5
+ (the adapter does); it turns raw readouts into distributions, confidences and
6
+ usage, and it will own the permutation policy.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import time
12
+ from collections.abc import Callable, Sequence
13
+ from dataclasses import dataclass, field
14
+
15
+ from jevify.domain.distribution import Distribution
16
+ from jevify.domain.questions import (
17
+ Question,
18
+ QuestionKind,
19
+ ScoreQuestion,
20
+ Semantics,
21
+ State,
22
+ )
23
+ from jevify.ports.backend import Backend, RawAnswer, StateHandle
24
+ from jevify.recipes.schema import Recipe
25
+
26
+ Clock = Callable[[], float]
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ReadoutRecord:
31
+ rung: str
32
+ degraded: bool
33
+ off_menu_mass: float | None
34
+ permutation: str
35
+ calls: int
36
+ latency_ms: float
37
+ prompt_tokens: int | None
38
+ cached_tokens: int | None
39
+ missing: tuple[str, ...] = ()
40
+ temperature: float | None = None
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class Answer:
45
+ question_id: str
46
+ kind: QuestionKind
47
+ distribution: Distribution
48
+ confidence: float
49
+ semantics: Semantics
50
+ readout: ReadoutRecord
51
+ legend: dict[str, str] = field(default_factory=dict)
52
+
53
+ @property
54
+ def selected(self) -> str:
55
+ return self.distribution.argmax()
56
+
57
+ @property
58
+ def expectation(self) -> float:
59
+ return self.distribution.expectation()
60
+
61
+ @property
62
+ def probability_true(self) -> float:
63
+ return self.distribution.as_mapping()["true"]
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Usage:
68
+ input_tokens: int
69
+ output_tokens: int
70
+
71
+
72
+ @dataclass(frozen=True)
73
+ class Evaluation:
74
+ state_id: str
75
+ answers: list[Answer]
76
+ usage: Usage
77
+ warm_ms: float | None
78
+ elapsed_ms: float
79
+
80
+
81
+ class Engine:
82
+ def __init__(
83
+ self, backend: Backend, recipe: Recipe, *, clock: Clock = time.perf_counter
84
+ ) -> None:
85
+ self.backend = backend
86
+ self.recipe = recipe
87
+ self.clock = clock
88
+
89
+ async def warm(self, state: State) -> StateHandle:
90
+ return await self.backend.warm(state)
91
+
92
+ async def evaluate(self, handle: StateHandle, questions: Sequence[Question]) -> Evaluation:
93
+ started = self.clock()
94
+ raw = await self.backend.evaluate(handle, questions)
95
+ by_id = {q.id: q for q in questions}
96
+ answers = [self._answer(by_id[r.question_id], r) for r in raw]
97
+ usage = Usage(
98
+ input_tokens=sum(r.prompt_tokens or 0 for r in raw),
99
+ output_tokens=len(raw),
100
+ )
101
+ return Evaluation(
102
+ state_id=handle.state_id,
103
+ answers=answers,
104
+ usage=usage,
105
+ warm_ms=handle.warm_ms,
106
+ elapsed_ms=(self.clock() - started) * 1000.0,
107
+ )
108
+
109
+ async def ask(self, state: State, questions: Sequence[Question]) -> Evaluation:
110
+ handle = await self.warm(state)
111
+ return await self.evaluate(handle, questions)
112
+
113
+ def _answer(self, question: Question, raw: RawAnswer) -> Answer:
114
+ ordered = {label: raw.logprobs[label] for label in question.labels if label in raw.logprobs}
115
+ temperature = self._temperature(question, raw)
116
+ if temperature is not None:
117
+ ordered = {label: value / temperature for label, value in ordered.items()}
118
+ distribution = Distribution.from_logprobs(ordered)
119
+ legend = (
120
+ {str(i): level for i, level in enumerate(question.levels)}
121
+ if isinstance(question, ScoreQuestion)
122
+ else {}
123
+ )
124
+ return Answer(
125
+ question_id=question.id,
126
+ kind=question.kind,
127
+ distribution=distribution,
128
+ confidence=distribution.confidence(question.kind),
129
+ semantics="calibrated" if temperature is not None else raw.semantics,
130
+ readout=ReadoutRecord(
131
+ rung=str(raw.rung),
132
+ degraded=raw.degraded,
133
+ off_menu_mass=raw.off_menu_mass,
134
+ permutation=self.recipe.readout.permutation,
135
+ calls=raw.calls,
136
+ latency_ms=raw.latency_ms,
137
+ prompt_tokens=raw.prompt_tokens,
138
+ cached_tokens=raw.cached_tokens,
139
+ missing=raw.missing,
140
+ temperature=temperature,
141
+ ),
142
+ legend=legend,
143
+ )
144
+
145
+ def _temperature(self, question: Question, raw: RawAnswer) -> float | None:
146
+ """A fitted temperature applies only to readouts, only for a bucket the table covers."""
147
+ table = self.recipe.calibration
148
+ if table is None or raw.semantics != "readout":
149
+ return None
150
+ from jevify.evaluation.calibrate import bucket_key
151
+
152
+ return table.temperatures.get(bucket_key(question.kind, len(question.labels)))
@@ -0,0 +1,121 @@
1
+ """Value types for state and typed questions (docs/00-invariants.md §2)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Mapping, Sequence
7
+ from dataclasses import dataclass, field
8
+ from typing import Literal
9
+
10
+ Semantics = Literal["readout", "relevance", "similarity", "calibrated"]
11
+ QuestionKind = Literal["choice", "score", "noul"]
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class TextPart:
16
+ text: str
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class ImagePart:
21
+ """An image as a data URI. jevify never fetches URLs (ADR-0003 D5)."""
22
+
23
+ data_uri: str
24
+
25
+
26
+ Part = TextPart | ImagePart
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class State:
31
+ """The shared input every question is asked about."""
32
+
33
+ parts: tuple[Part, ...]
34
+
35
+ @classmethod
36
+ def from_jev(cls, value: str | Mapping[str, object] | Sequence[object]) -> State:
37
+ """Build from Jev's `state` field: a string, a JSON object or array, or a list of
38
+ content parts in the chat-endpoint shape (`text` and `image_url` parts).
39
+
40
+ Structured values render deterministically as indented JSON with key order
41
+ preserved, so the same state always produces the same prefix. Images must be
42
+ data URIs: jevify never fetches a URL on a caller's behalf (ADR-0003 D5).
43
+ """
44
+ if isinstance(value, str):
45
+ return cls((TextPart(value),))
46
+ if isinstance(value, Sequence) and value and all(_is_content_part(v) for v in value):
47
+ parts: list[Part] = []
48
+ for item in value:
49
+ assert isinstance(item, Mapping)
50
+ if item["type"] == "text":
51
+ parts.append(TextPart(str(item.get("text", ""))))
52
+ else:
53
+ image = item.get("image_url")
54
+ url = image.get("url") if isinstance(image, Mapping) else image
55
+ if not isinstance(url, str) or not url.startswith("data:"):
56
+ raise ValueError(
57
+ "image parts must be data URIs; jevify never fetches a remote URL"
58
+ )
59
+ parts.append(ImagePart(url))
60
+ return cls(tuple(parts))
61
+ rendered = json.dumps(value, ensure_ascii=False, indent=2)
62
+ return cls((TextPart(rendered),))
63
+
64
+ @property
65
+ def text(self) -> str:
66
+ return "".join(part.text for part in self.parts if isinstance(part, TextPart))
67
+
68
+ @property
69
+ def modalities(self) -> frozenset[str]:
70
+ kinds = {"text" if isinstance(p, TextPart) else "image" for p in self.parts}
71
+ return frozenset(kinds)
72
+
73
+
74
+ def _is_content_part(value: object) -> bool:
75
+ return isinstance(value, Mapping) and value.get("type") in ("text", "image_url")
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class Option:
80
+ key: str
81
+ description: str | None = None
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class ChoiceQuestion:
86
+ id: str
87
+ instructions: str | None
88
+ options: tuple[Option, ...]
89
+ kind: QuestionKind = field(default="choice", init=False)
90
+
91
+ @property
92
+ def labels(self) -> tuple[str, ...]:
93
+ return tuple(option.key for option in self.options)
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class ScoreQuestion:
98
+ id: str
99
+ instructions: str | None
100
+ levels: tuple[str, ...]
101
+ kind: QuestionKind = field(default="score", init=False)
102
+
103
+ @property
104
+ def labels(self) -> tuple[str, ...]:
105
+ return tuple(str(index) for index in range(len(self.levels)))
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class NoulQuestion:
110
+ id: str
111
+ instructions: str | None
112
+ true_criterion: str | None = None
113
+ false_criterion: str | None = None
114
+ kind: QuestionKind = field(default="noul", init=False)
115
+
116
+ @property
117
+ def labels(self) -> tuple[str, ...]:
118
+ return ("false", "true")
119
+
120
+
121
+ Question = ChoiceQuestion | ScoreQuestion | NoulQuestion
@@ -0,0 +1 @@
1
+ """Evaluation (ADR-0005): frozen suites, metrics, scorecards, committed evidence."""
@@ -0,0 +1,186 @@
1
+ """Post-hoc calibration (ADR-0005 D5): one temperature per question type and option-count
2
+ bucket, fitted on labeled decisions, standard library only.
3
+
4
+ A temperature rescales log-probabilities before the softmax. It never changes a ranking; it
5
+ changes how sharp the distribution is. Fitting minimizes log loss on a fit split; the
6
+ evidence reports the held-out split before and after, so the table is judged on decisions
7
+ it never saw.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import json
14
+ import math
15
+ from collections.abc import Mapping, Sequence
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ from jevify.evaluation.metrics import brier_score, expected_calibration_error, log_loss
20
+
21
+ Row = tuple[Mapping[str, float], str] # (logprobs over labels, correct label)
22
+
23
+
24
+ def bucket_key(question_type: str, options: int) -> str:
25
+ if options <= 2:
26
+ band = "2"
27
+ elif options <= 5:
28
+ band = "3-5"
29
+ elif options <= 10:
30
+ band = "6-10"
31
+ else:
32
+ band = "11+"
33
+ return f"{question_type}:{band}"
34
+
35
+
36
+ def scale_logprobs(logprobs: Mapping[str, float], temperature: float) -> dict[str, float]:
37
+ return {label: value / temperature for label, value in logprobs.items()}
38
+
39
+
40
+ def _softmax(logprobs: Mapping[str, float]) -> dict[str, float]:
41
+ peak = max(logprobs.values())
42
+ weights = {k: math.exp(v - peak) for k, v in logprobs.items()}
43
+ total = sum(weights.values())
44
+ return {k: w / total for k, w in weights.items()}
45
+
46
+
47
+ def _loss(rows: Sequence[Row], temperature: float) -> float:
48
+ return log_loss(
49
+ [_softmax(scale_logprobs(lp, temperature)) for lp, _ in rows], [y for _, y in rows]
50
+ )
51
+
52
+
53
+ def fit_temperature(rows: Sequence[Row], *, low: float = 0.05, high: float = 20.0) -> float:
54
+ """Golden-section search over log-temperature for the minimum log loss."""
55
+ if not rows:
56
+ return 1.0
57
+ a, b = math.log(low), math.log(high)
58
+ phi = (math.sqrt(5) - 1) / 2
59
+ c = b - phi * (b - a)
60
+ d = a + phi * (b - a)
61
+ fc, fd = _loss(rows, math.exp(c)), _loss(rows, math.exp(d))
62
+ for _ in range(80):
63
+ if fc < fd:
64
+ b, d, fd = d, c, fc
65
+ c = b - phi * (b - a)
66
+ fc = _loss(rows, math.exp(c))
67
+ else:
68
+ a, c, fc = c, d, fd
69
+ d = a + phi * (b - a)
70
+ fd = _loss(rows, math.exp(d))
71
+ return math.exp((a + b) / 2)
72
+
73
+
74
+ def _rows_from_raw(raw_path: Path) -> list[dict[str, Any]]:
75
+ lines = [
76
+ json.loads(line)
77
+ for line in raw_path.read_text(encoding="utf-8").splitlines()
78
+ if line.strip()
79
+ ]
80
+ return [row for row in lines[1:] if row.get("label") is not None and not row.get("error")]
81
+
82
+
83
+ def _split(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
84
+ """Deterministic half split by case id hash, so a re-run reproduces the same split."""
85
+ fit, held = [], []
86
+ for row in rows:
87
+ digest = hashlib.sha256(row["case_id"].encode("utf-8")).digest()[0]
88
+ (fit if digest % 2 == 0 else held).append(row)
89
+ return fit, held
90
+
91
+
92
+ def _as_rows(decisions: list[dict[str, Any]]) -> list[Row]:
93
+ return [
94
+ ({k: math.log(max(v, 1e-12)) for k, v in d["probabilities"].items()}, d["label"])
95
+ for d in decisions
96
+ ]
97
+
98
+
99
+ def _metrics(decisions: list[dict[str, Any]], temperatures: Mapping[str, float]) -> dict[str, Any]:
100
+ probs, labels = [], []
101
+ for d in decisions:
102
+ key = bucket_key(d["question_type"], len(d["options"]))
103
+ lp = {k: math.log(max(v, 1e-12)) for k, v in d["probabilities"].items()}
104
+ probs.append(_softmax(scale_logprobs(lp, temperatures.get(key, 1.0))))
105
+ labels.append(d["label"])
106
+ if not labels:
107
+ return {}
108
+ ece, table = expected_calibration_error(probs, labels, bins=10)
109
+ return {
110
+ "decisions": len(labels),
111
+ "log_loss": log_loss(probs, labels),
112
+ "brier": brier_score(probs, labels),
113
+ "ece": ece,
114
+ "reliability": table,
115
+ }
116
+
117
+
118
+ def bucket_problems(buckets: Mapping[str, list[dict[str, Any]]], *, min_fit: int) -> list[str]:
119
+ """Why a fitted table must not be applied: too few decisions, or no error to learn from
120
+ (a fit without errors drives the temperature to the bound and claims certainty)."""
121
+ problems = []
122
+ for key, rows in buckets.items():
123
+ if len(rows) < min_fit:
124
+ problems.append(f"{key}: {len(rows)} fit decisions, fewer than {min_fit}")
125
+ if not any(max(d["probabilities"], key=d["probabilities"].get) != d["label"] for d in rows):
126
+ problems.append(f"{key}: no wrong decision in the fit split, nothing to calibrate")
127
+ return problems
128
+
129
+
130
+ def calibrate_run(raw_path: Path, *, recipe_hash_before: str, min_fit: int = 50) -> dict[str, Any]:
131
+ decisions = _rows_from_raw(raw_path)
132
+ fit, held = _split(decisions)
133
+ buckets: dict[str, list[dict[str, Any]]] = {}
134
+ for d in fit:
135
+ buckets.setdefault(bucket_key(d["question_type"], len(d["options"])), []).append(d)
136
+ temperatures = {key: fit_temperature(_as_rows(rows)) for key, rows in buckets.items()}
137
+ before = _metrics(held, {})
138
+ after = _metrics(held, temperatures)
139
+ problems = bucket_problems(buckets, min_fit=min_fit)
140
+ return {
141
+ "applicable": not problems,
142
+ "problems": problems,
143
+ "raw_file": raw_path.name,
144
+ "raw_sha256": hashlib.sha256(raw_path.read_bytes()).hexdigest(),
145
+ "recipe_hash_before": recipe_hash_before,
146
+ "split": {"fit": len(fit), "held_out": len(held), "rule": "sha256(case_id)[0] parity"},
147
+ "temperatures": temperatures,
148
+ "bucket_sizes": {k: len(v) for k, v in buckets.items()},
149
+ "held_out_before": {k: v for k, v in before.items() if k != "reliability"},
150
+ "held_out_after": {k: v for k, v in after.items() if k != "reliability"},
151
+ "reliability_before": before.get("reliability", []),
152
+ "reliability_after": after.get("reliability", []),
153
+ "note": (
154
+ "a temperature never changes a ranking; judge it on held-out log loss and ECE, and "
155
+ "on enough decisions: below a few hundred these numbers are indicative only"
156
+ ),
157
+ }
158
+
159
+
160
+ def render_markdown(evidence: dict[str, Any], recipe_stem: str) -> str:
161
+ lines = [
162
+ f"# Calibration: {recipe_stem}",
163
+ "",
164
+ f"Fitted on `runs/{evidence['raw_file']}` (sha256 `{evidence['raw_sha256'][:16]}…`), "
165
+ f"recipe hash before `{evidence['recipe_hash_before'][:16]}…`.",
166
+ "",
167
+ f"Split: {evidence['split']['fit']} decisions to fit, "
168
+ f"{evidence['split']['held_out']} held out ({evidence['split']['rule']}).",
169
+ "",
170
+ "| bucket | fit decisions | temperature |",
171
+ "|---|---|---|",
172
+ ]
173
+ for key, temperature in evidence["temperatures"].items():
174
+ lines.append(f"| {key} | {evidence['bucket_sizes'][key]} | {temperature:.3f} |")
175
+ lines += ["", "| held-out metric | before | after |", "|---|---|---|"]
176
+ for key in ("log_loss", "brier", "ece"):
177
+ before = evidence["held_out_before"].get(key)
178
+ after = evidence["held_out_after"].get(key)
179
+ if before is not None and after is not None:
180
+ lines.append(f"| {key} | {before:.4f} | {after:.4f} |")
181
+ if not evidence.get("applicable", True):
182
+ lines += ["", "**Not applicable as fitted:**"] + [f"- {p}" for p in evidence["problems"]]
183
+ if evidence.get("forced"):
184
+ lines.append("- the table was written anyway with --force; treat its label with care")
185
+ lines += ["", evidence["note"] + ".", ""]
186
+ return "\n".join(lines)
@@ -0,0 +1,78 @@
1
+ """Scorecard controls (ADR-0005 D3): the checks that make a main result interpretable.
2
+
3
+ - shuffled context: every case is answered against another case's state; agreement with the
4
+ original label should collapse if the answers depend on the state at all.
5
+ - option permutation: every choice case is asked again with its options reversed; the share
6
+ of unchanged selections measures position stability.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ from dataclasses import replace
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from jevify.domain.engine import Engine
17
+ from jevify.evaluation.scorecard import Decision, run_case
18
+ from jevify.evaluation.suites import Suite, SuiteRow
19
+
20
+
21
+ async def _run(
22
+ engine: Engine, rows: list[SuiteRow], concurrency: int, suite_path: Path
23
+ ) -> list[Decision]:
24
+ gate = asyncio.Semaphore(max(1, concurrency))
25
+
26
+ async def one(row: SuiteRow) -> Decision:
27
+ async with gate:
28
+ return await run_case(engine, row, suite_path)
29
+
30
+ return list(await asyncio.gather(*(one(row) for row in rows)))
31
+
32
+
33
+ async def shuffled_context(engine: Engine, suite: Suite, *, concurrency: int = 1) -> dict[str, Any]:
34
+ """Each case answered against the next case's state: its context text and its images."""
35
+ rows = list(suite.rows)
36
+ swapped = [
37
+ replace(
38
+ row,
39
+ context=rows[(index + 1) % len(rows)].context,
40
+ images=rows[(index + 1) % len(rows)].images,
41
+ )
42
+ for index, row in enumerate(rows)
43
+ ]
44
+ decisions = await _run(engine, swapped, concurrency, suite.path)
45
+ scored = [d for d in decisions if d.label is not None and not d.error]
46
+ agreement = sum(d.selected == d.label for d in scored) / max(1, len(scored))
47
+ return {
48
+ "cases": len(scored),
49
+ "agreement_with_label": agreement,
50
+ "failed": [{"case_id": d.case_id, "error": d.error} for d in decisions if d.error],
51
+ "note": "each case answered against the next case's state; low agreement means the "
52
+ "answers depend on the state",
53
+ }
54
+
55
+
56
+ async def option_permutation(
57
+ engine: Engine, suite: Suite, main: list[Decision], *, concurrency: int = 1
58
+ ) -> dict[str, Any]:
59
+ rows = [row for row in suite.rows if row.question_type == "choice"]
60
+ reversed_rows = [
61
+ replace(
62
+ row,
63
+ options=tuple(reversed(row.options)),
64
+ label=(len(row.options) - 1 - row.label) if row.label is not None else None,
65
+ )
66
+ for row in rows
67
+ ]
68
+ decisions = await _run(engine, reversed_rows, concurrency, suite.path)
69
+ by_id = {d.case_id: d for d in main}
70
+ pairs = [(by_id[d.case_id], d) for d in decisions if d.case_id in by_id and not d.error]
71
+ same = sum(original.selected == permuted.selected for original, permuted in pairs)
72
+ return {
73
+ "cases": len(pairs),
74
+ "same_choice": same / max(1, len(pairs)),
75
+ "failed": [{"case_id": d.case_id, "error": d.error} for d in decisions if d.error],
76
+ "note": "each choice case asked again with its options reversed; the share of "
77
+ "unchanged selections measures position stability",
78
+ }
@@ -0,0 +1,91 @@
1
+ """Metrics (ADR-0005 D4), standard library only. Every function takes plain lists so the
2
+ expected values in tests are literals, never a recomputation of the same code."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import math
7
+ from collections.abc import Mapping, Sequence
8
+
9
+ _EPSILON = 1e-12
10
+
11
+
12
+ def accuracy(selected: Sequence[str], labels: Sequence[str]) -> float:
13
+ if not labels:
14
+ return float("nan")
15
+ return sum(s == label for s, label in zip(selected, labels, strict=True)) / len(labels)
16
+
17
+
18
+ def log_loss(probabilities: Sequence[Mapping[str, float]], labels: Sequence[str]) -> float:
19
+ total = 0.0
20
+ for probs, label in zip(probabilities, labels, strict=True):
21
+ total -= math.log(max(probs.get(label, 0.0), _EPSILON))
22
+ return total / len(labels)
23
+
24
+
25
+ def brier_score(probabilities: Sequence[Mapping[str, float]], labels: Sequence[str]) -> float:
26
+ total = 0.0
27
+ for probs, label in zip(probabilities, labels, strict=True):
28
+ total += sum((p - (1.0 if key == label else 0.0)) ** 2 for key, p in probs.items())
29
+ return total / len(labels)
30
+
31
+
32
+ def expected_calibration_error(
33
+ probabilities: Sequence[Mapping[str, float]], labels: Sequence[str], *, bins: int = 10
34
+ ) -> tuple[float, list[dict[str, float | int]]]:
35
+ """ECE over equal-width confidence bins on the top probability, plus the reliability table.
36
+
37
+ Confidence 1.0 falls in the last bin. Empty bins contribute nothing.
38
+ """
39
+ table = [
40
+ {"lower": i / bins, "upper": (i + 1) / bins, "count": 0, "confidence": 0.0, "accuracy": 0.0}
41
+ for i in range(bins)
42
+ ]
43
+ for probs, label in zip(probabilities, labels, strict=True):
44
+ top_label = max(probs, key=lambda k: probs[k])
45
+ confidence = probs[top_label]
46
+ index = min(int(confidence * bins), bins - 1)
47
+ cell = table[index]
48
+ cell["count"] += 1
49
+ cell["confidence"] += confidence
50
+ cell["accuracy"] += 1.0 if top_label == label else 0.0
51
+ total = len(labels)
52
+ ece = 0.0
53
+ for cell in table:
54
+ if cell["count"]:
55
+ cell["confidence"] /= cell["count"]
56
+ cell["accuracy"] /= cell["count"]
57
+ ece += cell["count"] / total * abs(cell["accuracy"] - cell["confidence"])
58
+ return ece, table
59
+
60
+
61
+ def mean_absolute_error(expectations: Sequence[float], labels: Sequence[int]) -> float:
62
+ return sum(abs(e - label) for e, label in zip(expectations, labels, strict=True)) / len(labels)
63
+
64
+
65
+ def ranked_probability_score(
66
+ distributions: Sequence[Sequence[float]], labels: Sequence[int]
67
+ ) -> float:
68
+ """Mean RPS over ordered levels: squared distance between cumulative distributions,
69
+ divided by the number of levels minus one."""
70
+ total = 0.0
71
+ for probs, label in zip(distributions, labels, strict=True):
72
+ k = len(probs)
73
+ cdf_p = 0.0
74
+ cdf_t = 0.0
75
+ score = 0.0
76
+ for index, p in enumerate(probs):
77
+ cdf_p += p
78
+ cdf_t += 1.0 if index == label else 0.0
79
+ score += (cdf_p - cdf_t) ** 2
80
+ total += score / max(1, k - 1)
81
+ return total / len(labels)
82
+
83
+
84
+ def quantile(values: Sequence[float], q: float) -> float:
85
+ if not values:
86
+ return float("nan")
87
+ ordered = sorted(values)
88
+ position = (len(ordered) - 1) * q
89
+ low = int(position)
90
+ high = min(low + 1, len(ordered) - 1)
91
+ return ordered[low] + (ordered[high] - ordered[low]) * (position - low)