jevify 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jevify/__init__.py +7 -0
- jevify/__main__.py +9 -0
- jevify/adapters/__init__.py +1 -0
- jevify/adapters/embedding/__init__.py +1 -0
- jevify/adapters/embedding/adapter.py +203 -0
- jevify/adapters/embedding/local.py +82 -0
- jevify/adapters/encoder/__init__.py +1 -0
- jevify/adapters/encoder/adapter.py +165 -0
- jevify/adapters/encoder/local.py +77 -0
- jevify/adapters/endpoint/__init__.py +1 -0
- jevify/adapters/endpoint/adapter.py +281 -0
- jevify/adapters/endpoint/client.py +114 -0
- jevify/adapters/endpoint/dialects.py +166 -0
- jevify/adapters/endpoint/probe.py +589 -0
- jevify/adapters/endpoint/readout.py +244 -0
- jevify/adapters/rerank/__init__.py +1 -0
- jevify/adapters/rerank/adapter.py +152 -0
- jevify/api/__init__.py +1 -0
- jevify/api/app.py +228 -0
- jevify/api/schema.py +123 -0
- jevify/api/translate.py +239 -0
- jevify/cli.py +392 -0
- jevify/compose.py +82 -0
- jevify/domain/__init__.py +1 -0
- jevify/domain/distribution.py +103 -0
- jevify/domain/engine.py +152 -0
- jevify/domain/questions.py +121 -0
- jevify/evaluation/__init__.py +1 -0
- jevify/evaluation/calibrate.py +186 -0
- jevify/evaluation/controls.py +78 -0
- jevify/evaluation/metrics.py +91 -0
- jevify/evaluation/scorecard.py +385 -0
- jevify/evaluation/suites.py +166 -0
- jevify/ports/__init__.py +1 -0
- jevify/ports/backend.py +162 -0
- jevify/recipes/__init__.py +21 -0
- jevify/recipes/probe_case.yaml +72 -0
- jevify/recipes/render.py +198 -0
- jevify/recipes/schema.py +204 -0
- jevify/recipes/store.py +81 -0
- jevify-0.1.0.dist-info/METADATA +243 -0
- jevify-0.1.0.dist-info/RECORD +45 -0
- jevify-0.1.0.dist-info/WHEEL +4 -0
- jevify-0.1.0.dist-info/entry_points.txt +2 -0
- jevify-0.1.0.dist-info/licenses/LICENSE +21 -0
jevify/domain/engine.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""The Engine: warm a state, evaluate typed questions, return Answers with semantics.
|
|
2
|
+
|
|
3
|
+
The engine is the only caller of the backend port and the only producer of
|
|
4
|
+
`Answer`. It never renders prompts (the recipe does) and never reads logprobs
|
|
5
|
+
(the adapter does); it turns raw readouts into distributions, confidences and
|
|
6
|
+
usage, and it will own the permutation policy.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Callable, Sequence
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from jevify.domain.distribution import Distribution
|
|
16
|
+
from jevify.domain.questions import (
|
|
17
|
+
Question,
|
|
18
|
+
QuestionKind,
|
|
19
|
+
ScoreQuestion,
|
|
20
|
+
Semantics,
|
|
21
|
+
State,
|
|
22
|
+
)
|
|
23
|
+
from jevify.ports.backend import Backend, RawAnswer, StateHandle
|
|
24
|
+
from jevify.recipes.schema import Recipe
|
|
25
|
+
|
|
26
|
+
Clock = Callable[[], float]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class ReadoutRecord:
|
|
31
|
+
rung: str
|
|
32
|
+
degraded: bool
|
|
33
|
+
off_menu_mass: float | None
|
|
34
|
+
permutation: str
|
|
35
|
+
calls: int
|
|
36
|
+
latency_ms: float
|
|
37
|
+
prompt_tokens: int | None
|
|
38
|
+
cached_tokens: int | None
|
|
39
|
+
missing: tuple[str, ...] = ()
|
|
40
|
+
temperature: float | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Answer:
|
|
45
|
+
question_id: str
|
|
46
|
+
kind: QuestionKind
|
|
47
|
+
distribution: Distribution
|
|
48
|
+
confidence: float
|
|
49
|
+
semantics: Semantics
|
|
50
|
+
readout: ReadoutRecord
|
|
51
|
+
legend: dict[str, str] = field(default_factory=dict)
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def selected(self) -> str:
|
|
55
|
+
return self.distribution.argmax()
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def expectation(self) -> float:
|
|
59
|
+
return self.distribution.expectation()
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def probability_true(self) -> float:
|
|
63
|
+
return self.distribution.as_mapping()["true"]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Usage:
|
|
68
|
+
input_tokens: int
|
|
69
|
+
output_tokens: int
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class Evaluation:
|
|
74
|
+
state_id: str
|
|
75
|
+
answers: list[Answer]
|
|
76
|
+
usage: Usage
|
|
77
|
+
warm_ms: float | None
|
|
78
|
+
elapsed_ms: float
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Engine:
|
|
82
|
+
def __init__(
|
|
83
|
+
self, backend: Backend, recipe: Recipe, *, clock: Clock = time.perf_counter
|
|
84
|
+
) -> None:
|
|
85
|
+
self.backend = backend
|
|
86
|
+
self.recipe = recipe
|
|
87
|
+
self.clock = clock
|
|
88
|
+
|
|
89
|
+
async def warm(self, state: State) -> StateHandle:
|
|
90
|
+
return await self.backend.warm(state)
|
|
91
|
+
|
|
92
|
+
async def evaluate(self, handle: StateHandle, questions: Sequence[Question]) -> Evaluation:
|
|
93
|
+
started = self.clock()
|
|
94
|
+
raw = await self.backend.evaluate(handle, questions)
|
|
95
|
+
by_id = {q.id: q for q in questions}
|
|
96
|
+
answers = [self._answer(by_id[r.question_id], r) for r in raw]
|
|
97
|
+
usage = Usage(
|
|
98
|
+
input_tokens=sum(r.prompt_tokens or 0 for r in raw),
|
|
99
|
+
output_tokens=len(raw),
|
|
100
|
+
)
|
|
101
|
+
return Evaluation(
|
|
102
|
+
state_id=handle.state_id,
|
|
103
|
+
answers=answers,
|
|
104
|
+
usage=usage,
|
|
105
|
+
warm_ms=handle.warm_ms,
|
|
106
|
+
elapsed_ms=(self.clock() - started) * 1000.0,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
async def ask(self, state: State, questions: Sequence[Question]) -> Evaluation:
|
|
110
|
+
handle = await self.warm(state)
|
|
111
|
+
return await self.evaluate(handle, questions)
|
|
112
|
+
|
|
113
|
+
def _answer(self, question: Question, raw: RawAnswer) -> Answer:
|
|
114
|
+
ordered = {label: raw.logprobs[label] for label in question.labels if label in raw.logprobs}
|
|
115
|
+
temperature = self._temperature(question, raw)
|
|
116
|
+
if temperature is not None:
|
|
117
|
+
ordered = {label: value / temperature for label, value in ordered.items()}
|
|
118
|
+
distribution = Distribution.from_logprobs(ordered)
|
|
119
|
+
legend = (
|
|
120
|
+
{str(i): level for i, level in enumerate(question.levels)}
|
|
121
|
+
if isinstance(question, ScoreQuestion)
|
|
122
|
+
else {}
|
|
123
|
+
)
|
|
124
|
+
return Answer(
|
|
125
|
+
question_id=question.id,
|
|
126
|
+
kind=question.kind,
|
|
127
|
+
distribution=distribution,
|
|
128
|
+
confidence=distribution.confidence(question.kind),
|
|
129
|
+
semantics="calibrated" if temperature is not None else raw.semantics,
|
|
130
|
+
readout=ReadoutRecord(
|
|
131
|
+
rung=str(raw.rung),
|
|
132
|
+
degraded=raw.degraded,
|
|
133
|
+
off_menu_mass=raw.off_menu_mass,
|
|
134
|
+
permutation=self.recipe.readout.permutation,
|
|
135
|
+
calls=raw.calls,
|
|
136
|
+
latency_ms=raw.latency_ms,
|
|
137
|
+
prompt_tokens=raw.prompt_tokens,
|
|
138
|
+
cached_tokens=raw.cached_tokens,
|
|
139
|
+
missing=raw.missing,
|
|
140
|
+
temperature=temperature,
|
|
141
|
+
),
|
|
142
|
+
legend=legend,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
def _temperature(self, question: Question, raw: RawAnswer) -> float | None:
|
|
146
|
+
"""A fitted temperature applies only to readouts, only for a bucket the table covers."""
|
|
147
|
+
table = self.recipe.calibration
|
|
148
|
+
if table is None or raw.semantics != "readout":
|
|
149
|
+
return None
|
|
150
|
+
from jevify.evaluation.calibrate import bucket_key
|
|
151
|
+
|
|
152
|
+
return table.temperatures.get(bucket_key(question.kind, len(question.labels)))
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Value types for state and typed questions (docs/00-invariants.md §2)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
Semantics = Literal["readout", "relevance", "similarity", "calibrated"]
|
|
11
|
+
QuestionKind = Literal["choice", "score", "noul"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class TextPart:
|
|
16
|
+
text: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ImagePart:
|
|
21
|
+
"""An image as a data URI. jevify never fetches URLs (ADR-0003 D5)."""
|
|
22
|
+
|
|
23
|
+
data_uri: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
Part = TextPart | ImagePart
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class State:
|
|
31
|
+
"""The shared input every question is asked about."""
|
|
32
|
+
|
|
33
|
+
parts: tuple[Part, ...]
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_jev(cls, value: str | Mapping[str, object] | Sequence[object]) -> State:
|
|
37
|
+
"""Build from Jev's `state` field: a string, a JSON object or array, or a list of
|
|
38
|
+
content parts in the chat-endpoint shape (`text` and `image_url` parts).
|
|
39
|
+
|
|
40
|
+
Structured values render deterministically as indented JSON with key order
|
|
41
|
+
preserved, so the same state always produces the same prefix. Images must be
|
|
42
|
+
data URIs: jevify never fetches a URL on a caller's behalf (ADR-0003 D5).
|
|
43
|
+
"""
|
|
44
|
+
if isinstance(value, str):
|
|
45
|
+
return cls((TextPart(value),))
|
|
46
|
+
if isinstance(value, Sequence) and value and all(_is_content_part(v) for v in value):
|
|
47
|
+
parts: list[Part] = []
|
|
48
|
+
for item in value:
|
|
49
|
+
assert isinstance(item, Mapping)
|
|
50
|
+
if item["type"] == "text":
|
|
51
|
+
parts.append(TextPart(str(item.get("text", ""))))
|
|
52
|
+
else:
|
|
53
|
+
image = item.get("image_url")
|
|
54
|
+
url = image.get("url") if isinstance(image, Mapping) else image
|
|
55
|
+
if not isinstance(url, str) or not url.startswith("data:"):
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"image parts must be data URIs; jevify never fetches a remote URL"
|
|
58
|
+
)
|
|
59
|
+
parts.append(ImagePart(url))
|
|
60
|
+
return cls(tuple(parts))
|
|
61
|
+
rendered = json.dumps(value, ensure_ascii=False, indent=2)
|
|
62
|
+
return cls((TextPart(rendered),))
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def text(self) -> str:
|
|
66
|
+
return "".join(part.text for part in self.parts if isinstance(part, TextPart))
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def modalities(self) -> frozenset[str]:
|
|
70
|
+
kinds = {"text" if isinstance(p, TextPart) else "image" for p in self.parts}
|
|
71
|
+
return frozenset(kinds)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _is_content_part(value: object) -> bool:
|
|
75
|
+
return isinstance(value, Mapping) and value.get("type") in ("text", "image_url")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class Option:
|
|
80
|
+
key: str
|
|
81
|
+
description: str | None = None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class ChoiceQuestion:
|
|
86
|
+
id: str
|
|
87
|
+
instructions: str | None
|
|
88
|
+
options: tuple[Option, ...]
|
|
89
|
+
kind: QuestionKind = field(default="choice", init=False)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def labels(self) -> tuple[str, ...]:
|
|
93
|
+
return tuple(option.key for option in self.options)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class ScoreQuestion:
|
|
98
|
+
id: str
|
|
99
|
+
instructions: str | None
|
|
100
|
+
levels: tuple[str, ...]
|
|
101
|
+
kind: QuestionKind = field(default="score", init=False)
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def labels(self) -> tuple[str, ...]:
|
|
105
|
+
return tuple(str(index) for index in range(len(self.levels)))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(frozen=True)
|
|
109
|
+
class NoulQuestion:
|
|
110
|
+
id: str
|
|
111
|
+
instructions: str | None
|
|
112
|
+
true_criterion: str | None = None
|
|
113
|
+
false_criterion: str | None = None
|
|
114
|
+
kind: QuestionKind = field(default="noul", init=False)
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def labels(self) -> tuple[str, ...]:
|
|
118
|
+
return ("false", "true")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
Question = ChoiceQuestion | ScoreQuestion | NoulQuestion
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Evaluation (ADR-0005): frozen suites, metrics, scorecards, committed evidence."""
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Post-hoc calibration (ADR-0005 D5): one temperature per question type and option-count
|
|
2
|
+
bucket, fitted on labeled decisions, standard library only.
|
|
3
|
+
|
|
4
|
+
A temperature rescales log-probabilities before the softmax. It never changes a ranking; it
|
|
5
|
+
changes how sharp the distribution is. Fitting minimizes log loss on a fit split; the
|
|
6
|
+
evidence reports the held-out split before and after, so the table is judged on decisions
|
|
7
|
+
it never saw.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
from collections.abc import Mapping, Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from jevify.evaluation.metrics import brier_score, expected_calibration_error, log_loss
|
|
20
|
+
|
|
21
|
+
Row = tuple[Mapping[str, float], str] # (logprobs over labels, correct label)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def bucket_key(question_type: str, options: int) -> str:
|
|
25
|
+
if options <= 2:
|
|
26
|
+
band = "2"
|
|
27
|
+
elif options <= 5:
|
|
28
|
+
band = "3-5"
|
|
29
|
+
elif options <= 10:
|
|
30
|
+
band = "6-10"
|
|
31
|
+
else:
|
|
32
|
+
band = "11+"
|
|
33
|
+
return f"{question_type}:{band}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def scale_logprobs(logprobs: Mapping[str, float], temperature: float) -> dict[str, float]:
|
|
37
|
+
return {label: value / temperature for label, value in logprobs.items()}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _softmax(logprobs: Mapping[str, float]) -> dict[str, float]:
|
|
41
|
+
peak = max(logprobs.values())
|
|
42
|
+
weights = {k: math.exp(v - peak) for k, v in logprobs.items()}
|
|
43
|
+
total = sum(weights.values())
|
|
44
|
+
return {k: w / total for k, w in weights.items()}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _loss(rows: Sequence[Row], temperature: float) -> float:
|
|
48
|
+
return log_loss(
|
|
49
|
+
[_softmax(scale_logprobs(lp, temperature)) for lp, _ in rows], [y for _, y in rows]
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def fit_temperature(rows: Sequence[Row], *, low: float = 0.05, high: float = 20.0) -> float:
|
|
54
|
+
"""Golden-section search over log-temperature for the minimum log loss."""
|
|
55
|
+
if not rows:
|
|
56
|
+
return 1.0
|
|
57
|
+
a, b = math.log(low), math.log(high)
|
|
58
|
+
phi = (math.sqrt(5) - 1) / 2
|
|
59
|
+
c = b - phi * (b - a)
|
|
60
|
+
d = a + phi * (b - a)
|
|
61
|
+
fc, fd = _loss(rows, math.exp(c)), _loss(rows, math.exp(d))
|
|
62
|
+
for _ in range(80):
|
|
63
|
+
if fc < fd:
|
|
64
|
+
b, d, fd = d, c, fc
|
|
65
|
+
c = b - phi * (b - a)
|
|
66
|
+
fc = _loss(rows, math.exp(c))
|
|
67
|
+
else:
|
|
68
|
+
a, c, fc = c, d, fd
|
|
69
|
+
d = a + phi * (b - a)
|
|
70
|
+
fd = _loss(rows, math.exp(d))
|
|
71
|
+
return math.exp((a + b) / 2)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _rows_from_raw(raw_path: Path) -> list[dict[str, Any]]:
|
|
75
|
+
lines = [
|
|
76
|
+
json.loads(line)
|
|
77
|
+
for line in raw_path.read_text(encoding="utf-8").splitlines()
|
|
78
|
+
if line.strip()
|
|
79
|
+
]
|
|
80
|
+
return [row for row in lines[1:] if row.get("label") is not None and not row.get("error")]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _split(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
84
|
+
"""Deterministic half split by case id hash, so a re-run reproduces the same split."""
|
|
85
|
+
fit, held = [], []
|
|
86
|
+
for row in rows:
|
|
87
|
+
digest = hashlib.sha256(row["case_id"].encode("utf-8")).digest()[0]
|
|
88
|
+
(fit if digest % 2 == 0 else held).append(row)
|
|
89
|
+
return fit, held
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _as_rows(decisions: list[dict[str, Any]]) -> list[Row]:
|
|
93
|
+
return [
|
|
94
|
+
({k: math.log(max(v, 1e-12)) for k, v in d["probabilities"].items()}, d["label"])
|
|
95
|
+
for d in decisions
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _metrics(decisions: list[dict[str, Any]], temperatures: Mapping[str, float]) -> dict[str, Any]:
|
|
100
|
+
probs, labels = [], []
|
|
101
|
+
for d in decisions:
|
|
102
|
+
key = bucket_key(d["question_type"], len(d["options"]))
|
|
103
|
+
lp = {k: math.log(max(v, 1e-12)) for k, v in d["probabilities"].items()}
|
|
104
|
+
probs.append(_softmax(scale_logprobs(lp, temperatures.get(key, 1.0))))
|
|
105
|
+
labels.append(d["label"])
|
|
106
|
+
if not labels:
|
|
107
|
+
return {}
|
|
108
|
+
ece, table = expected_calibration_error(probs, labels, bins=10)
|
|
109
|
+
return {
|
|
110
|
+
"decisions": len(labels),
|
|
111
|
+
"log_loss": log_loss(probs, labels),
|
|
112
|
+
"brier": brier_score(probs, labels),
|
|
113
|
+
"ece": ece,
|
|
114
|
+
"reliability": table,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def bucket_problems(buckets: Mapping[str, list[dict[str, Any]]], *, min_fit: int) -> list[str]:
|
|
119
|
+
"""Why a fitted table must not be applied: too few decisions, or no error to learn from
|
|
120
|
+
(a fit without errors drives the temperature to the bound and claims certainty)."""
|
|
121
|
+
problems = []
|
|
122
|
+
for key, rows in buckets.items():
|
|
123
|
+
if len(rows) < min_fit:
|
|
124
|
+
problems.append(f"{key}: {len(rows)} fit decisions, fewer than {min_fit}")
|
|
125
|
+
if not any(max(d["probabilities"], key=d["probabilities"].get) != d["label"] for d in rows):
|
|
126
|
+
problems.append(f"{key}: no wrong decision in the fit split, nothing to calibrate")
|
|
127
|
+
return problems
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def calibrate_run(raw_path: Path, *, recipe_hash_before: str, min_fit: int = 50) -> dict[str, Any]:
|
|
131
|
+
decisions = _rows_from_raw(raw_path)
|
|
132
|
+
fit, held = _split(decisions)
|
|
133
|
+
buckets: dict[str, list[dict[str, Any]]] = {}
|
|
134
|
+
for d in fit:
|
|
135
|
+
buckets.setdefault(bucket_key(d["question_type"], len(d["options"])), []).append(d)
|
|
136
|
+
temperatures = {key: fit_temperature(_as_rows(rows)) for key, rows in buckets.items()}
|
|
137
|
+
before = _metrics(held, {})
|
|
138
|
+
after = _metrics(held, temperatures)
|
|
139
|
+
problems = bucket_problems(buckets, min_fit=min_fit)
|
|
140
|
+
return {
|
|
141
|
+
"applicable": not problems,
|
|
142
|
+
"problems": problems,
|
|
143
|
+
"raw_file": raw_path.name,
|
|
144
|
+
"raw_sha256": hashlib.sha256(raw_path.read_bytes()).hexdigest(),
|
|
145
|
+
"recipe_hash_before": recipe_hash_before,
|
|
146
|
+
"split": {"fit": len(fit), "held_out": len(held), "rule": "sha256(case_id)[0] parity"},
|
|
147
|
+
"temperatures": temperatures,
|
|
148
|
+
"bucket_sizes": {k: len(v) for k, v in buckets.items()},
|
|
149
|
+
"held_out_before": {k: v for k, v in before.items() if k != "reliability"},
|
|
150
|
+
"held_out_after": {k: v for k, v in after.items() if k != "reliability"},
|
|
151
|
+
"reliability_before": before.get("reliability", []),
|
|
152
|
+
"reliability_after": after.get("reliability", []),
|
|
153
|
+
"note": (
|
|
154
|
+
"a temperature never changes a ranking; judge it on held-out log loss and ECE, and "
|
|
155
|
+
"on enough decisions: below a few hundred these numbers are indicative only"
|
|
156
|
+
),
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def render_markdown(evidence: dict[str, Any], recipe_stem: str) -> str:
|
|
161
|
+
lines = [
|
|
162
|
+
f"# Calibration: {recipe_stem}",
|
|
163
|
+
"",
|
|
164
|
+
f"Fitted on `runs/{evidence['raw_file']}` (sha256 `{evidence['raw_sha256'][:16]}…`), "
|
|
165
|
+
f"recipe hash before `{evidence['recipe_hash_before'][:16]}…`.",
|
|
166
|
+
"",
|
|
167
|
+
f"Split: {evidence['split']['fit']} decisions to fit, "
|
|
168
|
+
f"{evidence['split']['held_out']} held out ({evidence['split']['rule']}).",
|
|
169
|
+
"",
|
|
170
|
+
"| bucket | fit decisions | temperature |",
|
|
171
|
+
"|---|---|---|",
|
|
172
|
+
]
|
|
173
|
+
for key, temperature in evidence["temperatures"].items():
|
|
174
|
+
lines.append(f"| {key} | {evidence['bucket_sizes'][key]} | {temperature:.3f} |")
|
|
175
|
+
lines += ["", "| held-out metric | before | after |", "|---|---|---|"]
|
|
176
|
+
for key in ("log_loss", "brier", "ece"):
|
|
177
|
+
before = evidence["held_out_before"].get(key)
|
|
178
|
+
after = evidence["held_out_after"].get(key)
|
|
179
|
+
if before is not None and after is not None:
|
|
180
|
+
lines.append(f"| {key} | {before:.4f} | {after:.4f} |")
|
|
181
|
+
if not evidence.get("applicable", True):
|
|
182
|
+
lines += ["", "**Not applicable as fitted:**"] + [f"- {p}" for p in evidence["problems"]]
|
|
183
|
+
if evidence.get("forced"):
|
|
184
|
+
lines.append("- the table was written anyway with --force; treat its label with care")
|
|
185
|
+
lines += ["", evidence["note"] + ".", ""]
|
|
186
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Scorecard controls (ADR-0005 D3): the checks that make a main result interpretable.
|
|
2
|
+
|
|
3
|
+
- shuffled context: every case is answered against another case's state; agreement with the
|
|
4
|
+
original label should collapse if the answers depend on the state at all.
|
|
5
|
+
- option permutation: every choice case is asked again with its options reversed; the share
|
|
6
|
+
of unchanged selections measures position stability.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
from dataclasses import replace
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from jevify.domain.engine import Engine
|
|
17
|
+
from jevify.evaluation.scorecard import Decision, run_case
|
|
18
|
+
from jevify.evaluation.suites import Suite, SuiteRow
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def _run(
|
|
22
|
+
engine: Engine, rows: list[SuiteRow], concurrency: int, suite_path: Path
|
|
23
|
+
) -> list[Decision]:
|
|
24
|
+
gate = asyncio.Semaphore(max(1, concurrency))
|
|
25
|
+
|
|
26
|
+
async def one(row: SuiteRow) -> Decision:
|
|
27
|
+
async with gate:
|
|
28
|
+
return await run_case(engine, row, suite_path)
|
|
29
|
+
|
|
30
|
+
return list(await asyncio.gather(*(one(row) for row in rows)))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def shuffled_context(engine: Engine, suite: Suite, *, concurrency: int = 1) -> dict[str, Any]:
|
|
34
|
+
"""Each case answered against the next case's state: its context text and its images."""
|
|
35
|
+
rows = list(suite.rows)
|
|
36
|
+
swapped = [
|
|
37
|
+
replace(
|
|
38
|
+
row,
|
|
39
|
+
context=rows[(index + 1) % len(rows)].context,
|
|
40
|
+
images=rows[(index + 1) % len(rows)].images,
|
|
41
|
+
)
|
|
42
|
+
for index, row in enumerate(rows)
|
|
43
|
+
]
|
|
44
|
+
decisions = await _run(engine, swapped, concurrency, suite.path)
|
|
45
|
+
scored = [d for d in decisions if d.label is not None and not d.error]
|
|
46
|
+
agreement = sum(d.selected == d.label for d in scored) / max(1, len(scored))
|
|
47
|
+
return {
|
|
48
|
+
"cases": len(scored),
|
|
49
|
+
"agreement_with_label": agreement,
|
|
50
|
+
"failed": [{"case_id": d.case_id, "error": d.error} for d in decisions if d.error],
|
|
51
|
+
"note": "each case answered against the next case's state; low agreement means the "
|
|
52
|
+
"answers depend on the state",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
async def option_permutation(
|
|
57
|
+
engine: Engine, suite: Suite, main: list[Decision], *, concurrency: int = 1
|
|
58
|
+
) -> dict[str, Any]:
|
|
59
|
+
rows = [row for row in suite.rows if row.question_type == "choice"]
|
|
60
|
+
reversed_rows = [
|
|
61
|
+
replace(
|
|
62
|
+
row,
|
|
63
|
+
options=tuple(reversed(row.options)),
|
|
64
|
+
label=(len(row.options) - 1 - row.label) if row.label is not None else None,
|
|
65
|
+
)
|
|
66
|
+
for row in rows
|
|
67
|
+
]
|
|
68
|
+
decisions = await _run(engine, reversed_rows, concurrency, suite.path)
|
|
69
|
+
by_id = {d.case_id: d for d in main}
|
|
70
|
+
pairs = [(by_id[d.case_id], d) for d in decisions if d.case_id in by_id and not d.error]
|
|
71
|
+
same = sum(original.selected == permuted.selected for original, permuted in pairs)
|
|
72
|
+
return {
|
|
73
|
+
"cases": len(pairs),
|
|
74
|
+
"same_choice": same / max(1, len(pairs)),
|
|
75
|
+
"failed": [{"case_id": d.case_id, "error": d.error} for d in decisions if d.error],
|
|
76
|
+
"note": "each choice case asked again with its options reversed; the share of "
|
|
77
|
+
"unchanged selections measures position stability",
|
|
78
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Metrics (ADR-0005 D4), standard library only. Every function takes plain lists so the
|
|
2
|
+
expected values in tests are literals, never a recomputation of the same code."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
from collections.abc import Mapping, Sequence
|
|
8
|
+
|
|
9
|
+
_EPSILON = 1e-12
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def accuracy(selected: Sequence[str], labels: Sequence[str]) -> float:
|
|
13
|
+
if not labels:
|
|
14
|
+
return float("nan")
|
|
15
|
+
return sum(s == label for s, label in zip(selected, labels, strict=True)) / len(labels)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def log_loss(probabilities: Sequence[Mapping[str, float]], labels: Sequence[str]) -> float:
|
|
19
|
+
total = 0.0
|
|
20
|
+
for probs, label in zip(probabilities, labels, strict=True):
|
|
21
|
+
total -= math.log(max(probs.get(label, 0.0), _EPSILON))
|
|
22
|
+
return total / len(labels)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def brier_score(probabilities: Sequence[Mapping[str, float]], labels: Sequence[str]) -> float:
|
|
26
|
+
total = 0.0
|
|
27
|
+
for probs, label in zip(probabilities, labels, strict=True):
|
|
28
|
+
total += sum((p - (1.0 if key == label else 0.0)) ** 2 for key, p in probs.items())
|
|
29
|
+
return total / len(labels)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def expected_calibration_error(
|
|
33
|
+
probabilities: Sequence[Mapping[str, float]], labels: Sequence[str], *, bins: int = 10
|
|
34
|
+
) -> tuple[float, list[dict[str, float | int]]]:
|
|
35
|
+
"""ECE over equal-width confidence bins on the top probability, plus the reliability table.
|
|
36
|
+
|
|
37
|
+
Confidence 1.0 falls in the last bin. Empty bins contribute nothing.
|
|
38
|
+
"""
|
|
39
|
+
table = [
|
|
40
|
+
{"lower": i / bins, "upper": (i + 1) / bins, "count": 0, "confidence": 0.0, "accuracy": 0.0}
|
|
41
|
+
for i in range(bins)
|
|
42
|
+
]
|
|
43
|
+
for probs, label in zip(probabilities, labels, strict=True):
|
|
44
|
+
top_label = max(probs, key=lambda k: probs[k])
|
|
45
|
+
confidence = probs[top_label]
|
|
46
|
+
index = min(int(confidence * bins), bins - 1)
|
|
47
|
+
cell = table[index]
|
|
48
|
+
cell["count"] += 1
|
|
49
|
+
cell["confidence"] += confidence
|
|
50
|
+
cell["accuracy"] += 1.0 if top_label == label else 0.0
|
|
51
|
+
total = len(labels)
|
|
52
|
+
ece = 0.0
|
|
53
|
+
for cell in table:
|
|
54
|
+
if cell["count"]:
|
|
55
|
+
cell["confidence"] /= cell["count"]
|
|
56
|
+
cell["accuracy"] /= cell["count"]
|
|
57
|
+
ece += cell["count"] / total * abs(cell["accuracy"] - cell["confidence"])
|
|
58
|
+
return ece, table
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def mean_absolute_error(expectations: Sequence[float], labels: Sequence[int]) -> float:
|
|
62
|
+
return sum(abs(e - label) for e, label in zip(expectations, labels, strict=True)) / len(labels)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def ranked_probability_score(
|
|
66
|
+
distributions: Sequence[Sequence[float]], labels: Sequence[int]
|
|
67
|
+
) -> float:
|
|
68
|
+
"""Mean RPS over ordered levels: squared distance between cumulative distributions,
|
|
69
|
+
divided by the number of levels minus one."""
|
|
70
|
+
total = 0.0
|
|
71
|
+
for probs, label in zip(distributions, labels, strict=True):
|
|
72
|
+
k = len(probs)
|
|
73
|
+
cdf_p = 0.0
|
|
74
|
+
cdf_t = 0.0
|
|
75
|
+
score = 0.0
|
|
76
|
+
for index, p in enumerate(probs):
|
|
77
|
+
cdf_p += p
|
|
78
|
+
cdf_t += 1.0 if index == label else 0.0
|
|
79
|
+
score += (cdf_p - cdf_t) ** 2
|
|
80
|
+
total += score / max(1, k - 1)
|
|
81
|
+
return total / len(labels)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def quantile(values: Sequence[float], q: float) -> float:
|
|
85
|
+
if not values:
|
|
86
|
+
return float("nan")
|
|
87
|
+
ordered = sorted(values)
|
|
88
|
+
position = (len(ordered) - 1) * q
|
|
89
|
+
low = int(position)
|
|
90
|
+
high = min(low + 1, len(ordered) - 1)
|
|
91
|
+
return ordered[low] + (ordered[high] - ordered[low]) * (position - low)
|