loopmath 0.4.3__py3-none-any.whl → 0.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/cli_registry.py +1 -0
- loopmath/eval/__init__.py +6 -0
- loopmath/eval/adapters/__init__.py +1 -0
- loopmath/eval/adapters/ocp.py +587 -0
- loopmath/eval/commands.py +237 -0
- loopmath/eval/contracts.py +1212 -0
- loopmath/eval/designcheck.py +503 -0
- loopmath/eval/engine_pymc.py +364 -0
- loopmath/eval/fit.py +279 -0
- loopmath/eval/fitfile.py +911 -0
- loopmath/eval/judges.py +997 -0
- loopmath/eval/labels.py +1268 -0
- loopmath/eval/model.py +700 -0
- loopmath/eval/quality.py +276 -0
- loopmath/eval/rubric.py +552 -0
- loopmath/eval/scores.py +902 -0
- loopmath/eval/simulate.py +1150 -0
- loopmath/eval/specs/__init__.py +38 -0
- loopmath/eval/specs/catalog.py +397 -0
- loopmath/eval/steps.py +334 -0
- loopmath/eval/testing.py +268 -0
- loopmath/eval/value.py +418 -0
- loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml +1 -1
- loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json +1 -1
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/METADATA +19 -2
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/RECORD +34 -13
- {loopmath-0.4.3.data → loopmath-0.4.4.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.4.3.data → loopmath-0.4.4.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.4.3.data → loopmath-0.4.4.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/WHEEL +0 -0
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/entry_points.txt +0 -0
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.4.3.dist-info → loopmath-0.4.4.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/cli_registry.py
CHANGED
|
@@ -1248,6 +1248,7 @@ def register(sub) -> None:
|
|
|
1248
1248
|
_add_tasks(sub)
|
|
1249
1249
|
_add_harness(sub)
|
|
1250
1250
|
_add_flow(sub)
|
|
1251
|
+
importlib.import_module("loopmath.eval.commands").add(sub) # `eval` (0.4.4, experimental)
|
|
1251
1252
|
|
|
1252
1253
|
|
|
1253
1254
|
# Handler targets, for the registry test: every one must import and be callable.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""`loopmath eval` (experimental): rubric scores from judges and reviewers, read as noisy measurements.
|
|
2
|
+
|
|
3
|
+
Each command is one stage: named files in, one file out, a versioned schema between stages. The package
|
|
4
|
+
imports nothing from `store`, `belief`, `flows`, `compare`, `recommend` or `gate`; only `adapters/ocp.py`
|
|
5
|
+
reads OCP documents, and only `engine_pymc.py` imports PyMC. `tests/eval/test_eval_import_rule.py` checks it.
|
|
6
|
+
"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Adapters between the score tables of `loopmath eval` and other loopmath documents."""
|
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
"""OCP signals, preferences and harness-runs rows to and from the score tables.
|
|
2
|
+
|
|
3
|
+
Not a command: `scores.py` calls it. The only module of `loopmath.eval` that may import `loopmath.ocp` and
|
|
4
|
+
`loopmath.harness_runs`.
|
|
5
|
+
|
|
6
|
+
The mapping, both ways:
|
|
7
|
+
|
|
8
|
+
- A reading is a score signal. `id` is the reading id, `name` the component, `source.ref` the judge, and
|
|
9
|
+
`ext["dev.loopmath.eval"]` holds `call`, `step`, `rubric`, `replaces`, `assignment`, `item` and `of`.
|
|
10
|
+
A signal's value is a number, so a level that is a word is written as `level` beside them, with the
|
|
11
|
+
value null. `cost_usd` and the members of the reading's own `ext` are kept there too.
|
|
12
|
+
- A pairwise judgement is a preference on a slate of runs of one input: `winner` is a run id or `tie`,
|
|
13
|
+
`members` is `[a, b]`, the judge id sits in `judge.ext["dev.loopmath.eval"].id`, and
|
|
14
|
+
`ext["dev.loopmath.eval"]` holds `component`, `step`, `call`, `shown_first`, `rubric`, `replaces` and
|
|
15
|
+
`assignment`.
|
|
16
|
+
- What an OCP record holds beyond these columns (a tier, a unit, an attempt, another namespace) is kept on
|
|
17
|
+
the study row under `ext["dev.loopmath.eval.ocp"]` and written back on export.
|
|
18
|
+
|
|
19
|
+
Records made by other tools have no `dev.loopmath.eval` member. For those, per (run, judge, name) the
|
|
20
|
+
newest score signal is the reading and each earlier one is a reading it corrects. A signal id of another
|
|
21
|
+
tool is unique in its run document, not in a study, so the reading's id and call are `<run id>.<signal
|
|
22
|
+
id>`: two documents that both hold a signal `s1` give two readings. The signal's own id is kept and
|
|
23
|
+
written back on export, where the eval member's `id` names the reading. A preference is read on the
|
|
24
|
+
overall component, or the one the caller names, with `shown_first` unknown.
|
|
25
|
+
|
|
26
|
+
Nothing here writes a file. `read_documents` and `read_rows` return rows for `scores.py` to validate and
|
|
27
|
+
merge; `export` returns what `eval scores export` prints.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import argparse
|
|
33
|
+
import copy
|
|
34
|
+
import datetime as _dt
|
|
35
|
+
import math
|
|
36
|
+
import re
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Any, Iterable, Mapping
|
|
40
|
+
|
|
41
|
+
from ...harness_runs.rows import read_numbered, run_id_for, stage_piece
|
|
42
|
+
from ...ocp.conformance import SIMULATE_KEY, validate_doc
|
|
43
|
+
from .. import contracts as C
|
|
44
|
+
|
|
45
|
+
NS = "dev.loopmath.eval"
|
|
46
|
+
KEPT = "dev.loopmath.eval.ocp"
|
|
47
|
+
READING_MEMBERS = ("call", "step", "rubric", "replaces", "assignment", "item", "of")
|
|
48
|
+
PAIR_MEMBERS = ("component", "step", "call", "shown_first", "rubric", "replaces", "assignment")
|
|
49
|
+
ALSO = ("level", "cost_usd", "observed_at", "ext") # written beside them when the row needs it
|
|
50
|
+
JUDGE_SOURCES = ("judge", "user")
|
|
51
|
+
_OCP_TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$")
|
|
52
|
+
_WORD = re.compile(r"[A-Za-z]+|\d+")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class Batch:
|
|
57
|
+
"""What one source gave: raw rows for the study tables, and every record that was not used.
|
|
58
|
+
|
|
59
|
+
`runs`, `scores` and `pairs` are not validated yet. `source` names where a row came from, by table and
|
|
60
|
+
id, for refusal messages. Each entry of `not_used` is `{"source", "kind", "id", "reason"}`."""
|
|
61
|
+
|
|
62
|
+
runs: list[dict] = field(default_factory=list)
|
|
63
|
+
scores: list[dict] = field(default_factory=list)
|
|
64
|
+
pairs: list[dict] = field(default_factory=list)
|
|
65
|
+
not_used: list[dict] = field(default_factory=list)
|
|
66
|
+
source: dict[tuple[str, str], str] = field(default_factory=dict)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run(args: argparse.Namespace) -> int:
|
|
70
|
+
"""The adapter is not a command. `eval scores` is the command that uses it."""
|
|
71
|
+
from ..scores import run as scores_run
|
|
72
|
+
|
|
73
|
+
return scores_run(args)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def family_of(model_id: str) -> str:
|
|
77
|
+
"""A family label for a model id that states none: its first word, lower case (`model-large` gives `model`).
|
|
78
|
+
|
|
79
|
+
A guess, used only when a record names no family. State `model.family` in the record to set it."""
|
|
80
|
+
words = _WORD.findall(model_id)
|
|
81
|
+
return words[0].lower() if words else "unknown"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def default_slate(input_id: str) -> str:
|
|
85
|
+
"""The slate id `export` gives the pairs of one input that came with no slate of their own."""
|
|
86
|
+
return f"eval.{input_id}"[:200]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _obj(value: Any) -> dict:
|
|
90
|
+
return value if isinstance(value, dict) else {}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _instant(value: Any) -> float:
|
|
94
|
+
try:
|
|
95
|
+
return _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
|
|
96
|
+
except (ValueError, OverflowError, OSError):
|
|
97
|
+
return 0.0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _total(costs: Iterable[Any]) -> float | None:
|
|
101
|
+
"""The sum of the costs that are numbers, or None when there is none or the sum is not a finite number."""
|
|
102
|
+
numbers = [x for x in costs if isinstance(x, (int, float)) and not isinstance(x, bool)]
|
|
103
|
+
try:
|
|
104
|
+
total = float(sum(numbers))
|
|
105
|
+
except OverflowError:
|
|
106
|
+
return None
|
|
107
|
+
return total if numbers and math.isfinite(total) else None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _ident(text: str) -> str:
|
|
111
|
+
return "".join("_" if ch.isspace() or ch in "/@" else ch for ch in text)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------------------------------------------------
|
|
115
|
+
# OCP run documents to rows
|
|
116
|
+
|
|
117
|
+
def _step(step: str, model: Any, effort: Any, ext: Any = None) -> dict:
|
|
118
|
+
ref = model if isinstance(model, dict) else {"id": model}
|
|
119
|
+
model_id = ref.get("id") or ref.get("raw") or "unknown"
|
|
120
|
+
family = ref.get("family") if isinstance(ref.get("family"), str) and ref.get("family") else family_of(model_id)
|
|
121
|
+
out: dict = {"step": step, "model": {"id": model_id, "family": _ident(family)}}
|
|
122
|
+
if isinstance(effort, str) and effort:
|
|
123
|
+
out["effort"] = effort
|
|
124
|
+
variant = _obj(_obj(ext).get(NS)).get("variant")
|
|
125
|
+
if isinstance(variant, str) and variant:
|
|
126
|
+
out["variant"] = variant
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def run_row(doc: dict, file: str) -> dict:
|
|
131
|
+
"""The study's run row of one OCP run document.
|
|
132
|
+
|
|
133
|
+
`input` is `run.task.id`. `arm` is `run.flow.arm`, else `run.labels.arm`, else the configuration id.
|
|
134
|
+
`steps` are the workflow's pieces in order with their settings; without a configuration, the last
|
|
135
|
+
attempt of each node. `cost_usd` is the sum of the attempts' `cost.usd`."""
|
|
136
|
+
run = _obj(doc.get("run"))
|
|
137
|
+
task, flow, config = _obj(run.get("task")), _obj(run.get("flow")), _obj(run.get("configuration"))
|
|
138
|
+
if not isinstance(run.get("id"), str) or not isinstance(task.get("id"), str):
|
|
139
|
+
raise C.ContractError(file, "run", "the document has no run.id or no run.task.id",
|
|
140
|
+
"give an OCP run document at version 0.3 or later, with its task", "no_run")
|
|
141
|
+
arm = flow.get("arm") or _obj(run.get("labels")).get("arm") or config.get("id")
|
|
142
|
+
if not isinstance(arm, str) or not arm:
|
|
143
|
+
raise C.ContractError(file, "run", f'run "{run["id"]}" names no arm',
|
|
144
|
+
"state run.flow.arm, run.labels.arm or a configuration id", "no_arm")
|
|
145
|
+
out: dict = {"run": run["id"], "input": task["id"], "arm": arm}
|
|
146
|
+
if isinstance(flow.get("split"), str) and flow["split"]:
|
|
147
|
+
out["split"] = flow["split"]
|
|
148
|
+
settings = _obj(config.get("settings"))
|
|
149
|
+
pieces = [p.get("id") for p in _obj(config.get("workflow")).get("pieces") or [] if isinstance(p, dict)]
|
|
150
|
+
order = [p for p in pieces if p in settings] + [p for p in settings if p not in pieces]
|
|
151
|
+
steps = [_step(_ident(p), _obj(settings[p]).get("model") or {}, _obj(settings[p]).get("effort"),
|
|
152
|
+
_obj(settings[p]).get("ext")) for p in order if isinstance(settings[p], dict)]
|
|
153
|
+
attempts = [a for a in doc.get("attempts") or [] if isinstance(a, dict)]
|
|
154
|
+
if not steps:
|
|
155
|
+
last: dict[str, dict] = {}
|
|
156
|
+
for attempt in attempts:
|
|
157
|
+
where = attempt.get("vertex") or attempt.get("node")
|
|
158
|
+
if isinstance(where, str) and where:
|
|
159
|
+
last.pop(where, None)
|
|
160
|
+
last[where] = attempt
|
|
161
|
+
steps = [_step(_ident(where), a.get("model") or {}, a.get("effort"), a.get("ext")) for where, a in last.items()]
|
|
162
|
+
if not steps:
|
|
163
|
+
raise C.ContractError(file, "run", f'run "{run["id"]}" has no step settings',
|
|
164
|
+
"give run.configuration.settings, or attempts with their models", "no_steps")
|
|
165
|
+
out["steps"] = steps
|
|
166
|
+
cost = _total(_obj(a.get("cost")).get("usd") for a in attempts)
|
|
167
|
+
if cost is not None:
|
|
168
|
+
out["cost_usd"] = cost
|
|
169
|
+
out["simulated"] = isinstance(_obj(run.get("ext")).get(SIMULATE_KEY), dict)
|
|
170
|
+
slate = run.get("slate")
|
|
171
|
+
if isinstance(slate, dict) and (slate.get("id") != default_slate(task["id"]) or set(slate) - {"id", "members"}):
|
|
172
|
+
out["ext"] = {KEPT: {"slate": copy.deepcopy(slate)}} # a slate `export` would not write by itself
|
|
173
|
+
return out
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _rest(record: dict, columns: Iterable[str], known: Iterable[str]) -> dict:
|
|
177
|
+
"""What an OCP record holds beyond the study row's columns, for `ext[KEPT]`."""
|
|
178
|
+
rest = {k: copy.deepcopy(v) for k, v in record.items() if k not in columns and k != "ext"}
|
|
179
|
+
if rest.get("tier") == "reported":
|
|
180
|
+
del rest["tier"]
|
|
181
|
+
ext = {k: copy.deepcopy(v) for k, v in _obj(record.get("ext")).items() if k != NS}
|
|
182
|
+
if ext:
|
|
183
|
+
rest["ext"] = ext
|
|
184
|
+
more = {k: copy.deepcopy(v) for k, v in _obj(_obj(record.get("ext")).get(NS)).items()
|
|
185
|
+
if k not in known and k not in ALSO}
|
|
186
|
+
if more:
|
|
187
|
+
rest["eval"] = more
|
|
188
|
+
return rest
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _common(row: dict, e: dict, names: Iterable[str], rest: dict) -> dict:
|
|
192
|
+
for name in names:
|
|
193
|
+
if e.get(name) is not None:
|
|
194
|
+
row[name] = e[name]
|
|
195
|
+
ext = dict(_obj(e.get("ext")))
|
|
196
|
+
if rest:
|
|
197
|
+
ext[KEPT] = rest
|
|
198
|
+
if ext:
|
|
199
|
+
row["ext"] = ext
|
|
200
|
+
return row
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def scoped_id(run_id: str, record_id: str) -> str:
|
|
204
|
+
"""The study-wide id of a record that another tool named within its run document."""
|
|
205
|
+
return _ident(f"{run_id}.{record_id}")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def reading_row(run_id: str, signal: dict, rubric_ref: str | None = None, *, scope: bool = False) -> dict:
|
|
209
|
+
"""The study's reading of one score signal. Not validated: `contracts.validate_score` does that.
|
|
210
|
+
|
|
211
|
+
The reading's id is the eval member's `id`, else the signal's id. With `scope`, a signal that has no
|
|
212
|
+
eval member is named within its run: its reading id and call are `scoped_id(run, signal id)`. When
|
|
213
|
+
the reading's id is not the signal's, the signal's id is kept for `signal` to write back."""
|
|
214
|
+
stated = _obj(signal.get("ext")).get(NS)
|
|
215
|
+
e = _obj(stated)
|
|
216
|
+
source = _obj(signal.get("source"))
|
|
217
|
+
given = signal.get("id")
|
|
218
|
+
ident = e.get("id") or given
|
|
219
|
+
if scope and not isinstance(stated, dict) and isinstance(given, str) and given:
|
|
220
|
+
ident = scoped_id(run_id, given)
|
|
221
|
+
row = {"id": ident, "run": run_id, "step": e.get("step") or C.FINAL, "component": signal.get("name"),
|
|
222
|
+
"judge": source.get("ref"), "call": e.get("call") or ident,
|
|
223
|
+
"value": e["level"] if e.get("level") is not None else signal.get("value"),
|
|
224
|
+
"rubric": e.get("rubric") or rubric_ref, "observed_at": e.get("observed_at") or signal.get("observed_at")}
|
|
225
|
+
rest = _rest(signal, ("id", "kind", "name", "value", "observed_at", "source"), (*READING_MEMBERS, "id"))
|
|
226
|
+
if ident != given:
|
|
227
|
+
rest["id"] = given
|
|
228
|
+
kept_source = {k: copy.deepcopy(v) for k, v in source.items() if k != "ref"}
|
|
229
|
+
if kept_source not in ({"kind": "judge"}, {"kind": "user"}): # which of the two: the judges file says
|
|
230
|
+
rest["source"] = kept_source
|
|
231
|
+
return _common(row, e, ("item", "of", "replaces", "assignment", "cost_usd"), rest)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _judge_id(judge: dict) -> str:
|
|
235
|
+
stated = _obj(_obj(judge.get("ext")).get(NS)).get("id")
|
|
236
|
+
if isinstance(stated, str) and stated:
|
|
237
|
+
return stated
|
|
238
|
+
return _ident(str(judge.get("model") or judge.get("kind") or "referee"))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def pair_row(preference: dict, members: list, inputs: Mapping[str, str], rubric_ref: str | None = None,
|
|
242
|
+
component: str | None = None, observed_at: str | None = None) -> dict:
|
|
243
|
+
"""The study's pair of one preference between the two runs `members`. Not validated."""
|
|
244
|
+
e = _obj(_obj(preference.get("ext")).get(NS))
|
|
245
|
+
a, b = members
|
|
246
|
+
judge = _obj(preference.get("judge"))
|
|
247
|
+
winner = preference.get("winner")
|
|
248
|
+
row = {"id": preference.get("id"), "a": a, "b": b, "step": e.get("step") or C.FINAL,
|
|
249
|
+
"component": e.get("component") or component, "judge": _judge_id(judge),
|
|
250
|
+
"call": e.get("call") or preference.get("id"), "shown_first": e.get("shown_first"),
|
|
251
|
+
"verdict": "a" if winner == a else "b" if winner == b else winner,
|
|
252
|
+
"rubric": e.get("rubric") or rubric_ref,
|
|
253
|
+
"observed_at": e.get("observed_at") or preference.get("observed_at") or observed_at}
|
|
254
|
+
rest = _rest(preference, ("id", "winner", "members", "observed_at", "judge", "slate"), PAIR_MEMBERS)
|
|
255
|
+
kept_judge = {k: copy.deepcopy(v) for k, v in judge.items() if k != "ext"}
|
|
256
|
+
judge_ext = {k: copy.deepcopy(v) for k, v in _obj(judge.get("ext")).items() if k != NS}
|
|
257
|
+
if judge_ext:
|
|
258
|
+
kept_judge["ext"] = judge_ext
|
|
259
|
+
if kept_judge and kept_judge not in ({"kind": "model"}, {"kind": "user"}):
|
|
260
|
+
rest["judge"] = kept_judge
|
|
261
|
+
if preference.get("slate") != default_slate(inputs.get(a, "")):
|
|
262
|
+
rest["slate"] = preference.get("slate")
|
|
263
|
+
return _common(row, e, ("replaces", "assignment", "cost_usd"), rest)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _signals(run_id: str, signals: Iterable[Any], where: str, batch: Batch, rubric_ref: str | None,
|
|
267
|
+
components: set[str] | None, *, scope: bool = False) -> None:
|
|
268
|
+
"""Add the readings of one run's signals to `batch`. Verdicts and events are not readings and are skipped.
|
|
269
|
+
|
|
270
|
+
`scope` is for run documents, whose signal ids are unique per document only (see `reading_row`)."""
|
|
271
|
+
plain: dict[tuple[str, Any], list[tuple[float, int, dict]]] = {}
|
|
272
|
+
|
|
273
|
+
def skip(signal: dict, reason: str) -> None:
|
|
274
|
+
batch.not_used.append({"source": where, "kind": "signal", "id": signal.get("id"), "reason": reason})
|
|
275
|
+
|
|
276
|
+
for n, signal in enumerate(signals):
|
|
277
|
+
if not isinstance(signal, dict) or signal.get("kind") != "score":
|
|
278
|
+
continue
|
|
279
|
+
source = _obj(signal.get("source"))
|
|
280
|
+
e = _obj(signal.get("ext")).get(NS)
|
|
281
|
+
if source.get("kind") not in JUDGE_SOURCES or not isinstance(source.get("ref"), str) or not source["ref"]:
|
|
282
|
+
skip(signal, "a score with no judge: source.kind is not judge or user, or source.ref is missing")
|
|
283
|
+
elif signal.get("value") is None and _obj(e).get("level") is None:
|
|
284
|
+
skip(signal, "a declared score with no value yet")
|
|
285
|
+
elif isinstance(e, dict):
|
|
286
|
+
row = reading_row(run_id, signal, rubric_ref, scope=scope)
|
|
287
|
+
batch.scores.append(row)
|
|
288
|
+
batch.source[("scores", str(row["id"]))] = where
|
|
289
|
+
elif rubric_ref is None:
|
|
290
|
+
skip(signal, "the signal names no rubric; give --rubric")
|
|
291
|
+
elif components is not None and signal.get("name") not in components:
|
|
292
|
+
skip(signal, f'"{signal.get("name")}" is not a component of the rubric')
|
|
293
|
+
else:
|
|
294
|
+
plain.setdefault((source["ref"], signal.get("name")), []).append(
|
|
295
|
+
(_instant(signal.get("observed_at")), n, signal))
|
|
296
|
+
for group in plain.values():
|
|
297
|
+
earlier = None
|
|
298
|
+
for _, _, signal in sorted(group, key=lambda entry: entry[:2]):
|
|
299
|
+
row = reading_row(run_id, signal, rubric_ref, scope=scope)
|
|
300
|
+
if earlier is not None:
|
|
301
|
+
row["replaces"] = earlier
|
|
302
|
+
earlier = row["id"]
|
|
303
|
+
batch.scores.append(row)
|
|
304
|
+
batch.source[("scores", str(row["id"]))] = where
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def read_documents(documents: Iterable[tuple[str, Any]], *, rubric: dict | None = None,
|
|
308
|
+
pair_component: str | None = None, inputs: Mapping[str, str] | None = None) -> Batch:
|
|
309
|
+
"""Runs, readings and pairs of OCP run documents, given as `(file name, parsed document)`.
|
|
310
|
+
|
|
311
|
+
A document that does not conform to OCP is a refusal that names the file and the place. `inputs` maps
|
|
312
|
+
the runs already in the study to their inputs, so a preference can name a run imported earlier."""
|
|
313
|
+
batch = Batch()
|
|
314
|
+
rubric_ref = None if rubric is None else C.rubric_ref(rubric)
|
|
315
|
+
components = None if rubric is None else {comp["id"] for comp in rubric["components"]}
|
|
316
|
+
known = dict(inputs or {})
|
|
317
|
+
docs = []
|
|
318
|
+
for file, doc in documents:
|
|
319
|
+
if not isinstance(doc, dict) or doc.get("kind") == "flow" or not isinstance(doc.get("run"), dict):
|
|
320
|
+
raise C.ContractError(file, "", "not an OCP run document", "give run documents, one per file", "no_run")
|
|
321
|
+
for finding in validate_doc(doc):
|
|
322
|
+
if finding.level == "error":
|
|
323
|
+
raise C.ContractError(file, finding.path, f"{finding.code}: {finding.message}",
|
|
324
|
+
"fix the document (loopmath ocp validate names every problem)", "bad_ocp")
|
|
325
|
+
row = run_row(doc, file)
|
|
326
|
+
batch.runs.append(row)
|
|
327
|
+
batch.source[("runs", row["run"])] = file
|
|
328
|
+
known[row["run"]] = row["input"]
|
|
329
|
+
docs.append((file, doc))
|
|
330
|
+
seen: dict[Any, dict] = {}
|
|
331
|
+
for file, doc in docs:
|
|
332
|
+
run = doc["run"]
|
|
333
|
+
_signals(run["id"], run.get("signals") or [], file, batch, rubric_ref, components, scope=True)
|
|
334
|
+
slate = _obj(run.get("slate"))
|
|
335
|
+
fallback_time = run.get("ended_at") or run.get("started_at") or _obj(doc.get("producer")).get("emitted_at")
|
|
336
|
+
for preference in run.get("preferences") or []:
|
|
337
|
+
if not isinstance(preference, dict):
|
|
338
|
+
continue
|
|
339
|
+
if seen.get(preference.get("id")) == preference:
|
|
340
|
+
continue # copied into every member's file: read once
|
|
341
|
+
seen[preference.get("id")] = preference
|
|
342
|
+
reason, row = None, None
|
|
343
|
+
has_eval = isinstance(_obj(preference.get("ext")).get(NS), dict)
|
|
344
|
+
members = preference.get("members")
|
|
345
|
+
if not isinstance(members, list) and preference.get("slate") == slate.get("id"):
|
|
346
|
+
members = slate.get("members")
|
|
347
|
+
if not isinstance(members, list) or len(members) != 2:
|
|
348
|
+
reason = "the preference does not name exactly two runs"
|
|
349
|
+
elif preference.get("winner") not in (*members, "tie"):
|
|
350
|
+
reason = "the winner is neither of the two runs nor tie"
|
|
351
|
+
elif not has_eval and (members[0] not in known or members[1] not in known):
|
|
352
|
+
reason = "the preference does not resolve to two runs of the study"
|
|
353
|
+
elif not has_eval and known[members[0]] != known[members[1]]:
|
|
354
|
+
reason = "the two runs have different inputs"
|
|
355
|
+
elif not has_eval and rubric_ref is None:
|
|
356
|
+
reason = "the preference names no rubric; give --rubric"
|
|
357
|
+
else:
|
|
358
|
+
row = pair_row(preference, members, known, rubric_ref, pair_component, fallback_time)
|
|
359
|
+
if row["observed_at"] is None:
|
|
360
|
+
reason = "the preference has no observed_at and its run has no time"
|
|
361
|
+
if reason is not None or row is None:
|
|
362
|
+
batch.not_used.append({"source": file, "kind": "preference", "id": preference.get("id"),
|
|
363
|
+
"reason": reason})
|
|
364
|
+
continue
|
|
365
|
+
batch.pairs.append(row)
|
|
366
|
+
batch.source[("pairs", str(row["id"]))] = file
|
|
367
|
+
return batch
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
# ---------------------------------------------------------------------------------------------------
|
|
371
|
+
# harness-runs rows to rows
|
|
372
|
+
|
|
373
|
+
def read_rows(path: Any, *, rubric: dict | None = None) -> Batch:
|
|
374
|
+
"""Runs and readings of a harness-runs file. A row that breaks its format is a refusal by line.
|
|
375
|
+
|
|
376
|
+
The run's `input` is `task_id`, its `arm` is `flow.arm` or `arm`, its steps are the stages (the last
|
|
377
|
+
stage of each piece), else one step `final` with the row's model. Each `outcome.judges` item is a
|
|
378
|
+
reading: `ext["dev.loopmath.eval"]` on the item states `component` and the members a signal holds. An item
|
|
379
|
+
without it is read on the rubric's overall component, or its only component. The row's own
|
|
380
|
+
`outcome.score` is a reading when the row names its `judge` and its `name` is a component."""
|
|
381
|
+
file = Path(path).name
|
|
382
|
+
try:
|
|
383
|
+
rows = read_numbered(path)
|
|
384
|
+
except ValueError as exc:
|
|
385
|
+
where, problem = "", str(exc)
|
|
386
|
+
if problem.startswith(f"{path}:"): # "<path>:<line>: <problem>"
|
|
387
|
+
line, _, rest = problem[len(f"{path}:"):].partition(": ")
|
|
388
|
+
where, problem = (f"line {line}", rest) if line.isdigit() and rest else ("", problem)
|
|
389
|
+
raise C.ContractError(file, where, problem, "fix the row, or check the path", "bad_rows") from None
|
|
390
|
+
batch = Batch()
|
|
391
|
+
rubric_ref = None if rubric is None else C.rubric_ref(rubric)
|
|
392
|
+
components = None if rubric is None else {comp["id"] for comp in rubric["components"]}
|
|
393
|
+
default = None
|
|
394
|
+
if rubric is not None:
|
|
395
|
+
default = C.overall_component(rubric) or (rubric["components"][0]["id"] if len(rubric["components"]) == 1
|
|
396
|
+
else None)
|
|
397
|
+
for line, row in rows:
|
|
398
|
+
where = f"{file} line {line}"
|
|
399
|
+
run_id = run_id_for(row)
|
|
400
|
+
flow, outcome = _obj(row.get("flow")), _obj(row.get("outcome"))
|
|
401
|
+
arm = flow.get("arm") or row.get("arm")
|
|
402
|
+
if not isinstance(arm, str) or not arm:
|
|
403
|
+
raise C.ContractError(file, f"line {line}", "the row names no arm", "state arm, or flow.arm", "no_arm")
|
|
404
|
+
steps: dict[str, dict] = {}
|
|
405
|
+
costs = [_obj(row.get("cost")).get("usd")]
|
|
406
|
+
for stage in row.get("stages") or []:
|
|
407
|
+
piece = _ident(stage_piece(stage)[0])
|
|
408
|
+
steps.pop(piece, None)
|
|
409
|
+
steps[piece] = _step(piece, stage.get("model") or row.get("model") or "unknown",
|
|
410
|
+
stage.get("effort") or row.get("effort"), stage.get("ext"))
|
|
411
|
+
costs.append(_obj(stage.get("cost")).get("usd"))
|
|
412
|
+
if not steps:
|
|
413
|
+
steps[C.FINAL] = _step(C.FINAL, row.get("model") or "unknown", row.get("effort"), row.get("ext"))
|
|
414
|
+
out: dict = {"run": run_id, "input": row["task_id"], "arm": arm}
|
|
415
|
+
if isinstance(flow.get("split"), str) and flow["split"]:
|
|
416
|
+
out["split"] = flow["split"]
|
|
417
|
+
out["steps"] = list(steps.values())
|
|
418
|
+
cost = _total(costs)
|
|
419
|
+
if cost is not None:
|
|
420
|
+
out["cost_usd"] = cost
|
|
421
|
+
out["simulated"] = isinstance(_obj(row.get("ext")).get(SIMULATE_KEY), dict)
|
|
422
|
+
batch.runs.append(out)
|
|
423
|
+
batch.source[("runs", run_id)] = where
|
|
424
|
+
at = outcome.get("observed_at") or row.get("ended_at") or row.get("started_at")
|
|
425
|
+
signals = []
|
|
426
|
+
if "score" in outcome and isinstance(outcome.get("judge"), str):
|
|
427
|
+
signals.append({"id": f"{run_id}:score", "kind": "score", "name": outcome.get("name") or "judge",
|
|
428
|
+
"value": outcome["score"], "observed_at": at,
|
|
429
|
+
"source": {"kind": "judge", "ref": outcome["judge"]}})
|
|
430
|
+
judges = outcome.get("judges") if isinstance(outcome.get("judges"), list) else []
|
|
431
|
+
for n, item in enumerate(judges):
|
|
432
|
+
if not isinstance(item, dict):
|
|
433
|
+
continue
|
|
434
|
+
e = _obj(_obj(item.get("ext")).get(NS))
|
|
435
|
+
name = e.get("component") or default
|
|
436
|
+
signal = {"id": e.get("id") or f"{run_id}:judge:{n}", "kind": "score", "name": name,
|
|
437
|
+
"value": item.get("score"), "observed_at": item.get("observed_at") or at,
|
|
438
|
+
"source": {"kind": "judge", "ref": item.get("id")}}
|
|
439
|
+
if e:
|
|
440
|
+
signal["ext"] = {NS: {k: v for k, v in e.items() if k not in ("component", "id")}}
|
|
441
|
+
reason = None
|
|
442
|
+
if name is None:
|
|
443
|
+
reason = ("the judge score names no component; give --rubric" if rubric is None else
|
|
444
|
+
"the judge score names no component, and the rubric has no overall component and "
|
|
445
|
+
"more than one component")
|
|
446
|
+
elif signal["observed_at"] is None and (signal["value"] is not None or e.get("level") is not None):
|
|
447
|
+
reason = "the judge score has no observed_at and its row has no time"
|
|
448
|
+
if reason is not None:
|
|
449
|
+
batch.not_used.append({"source": where, "kind": "signal", "id": signal["id"], "reason": reason})
|
|
450
|
+
continue
|
|
451
|
+
signals.append(signal)
|
|
452
|
+
_signals(run_id, signals, where, batch, rubric_ref, components)
|
|
453
|
+
return batch
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
# ---------------------------------------------------------------------------------------------------
|
|
457
|
+
# Rows to OCP signals and preferences
|
|
458
|
+
|
|
459
|
+
def _ocp_time(text: str) -> tuple[str, str | None]:
|
|
460
|
+
"""`(the time as OCP writes it, the text as given when the two differ)`."""
|
|
461
|
+
if _OCP_TIME.match(text):
|
|
462
|
+
return text, None
|
|
463
|
+
parsed = _dt.datetime.fromisoformat(text[:-1] + "+00:00" if text.endswith("Z") else text)
|
|
464
|
+
return parsed.isoformat(), text
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _eval_ext(row: dict, names: Iterable[str], kept: dict) -> tuple[dict, str]:
|
|
468
|
+
e: dict = {name: row[name] for name in names if row.get(name) is not None}
|
|
469
|
+
if row.get("cost_usd") is not None:
|
|
470
|
+
e["cost_usd"] = row["cost_usd"]
|
|
471
|
+
observed, raw = _ocp_time(row["observed_at"])
|
|
472
|
+
if raw is not None:
|
|
473
|
+
e["observed_at"] = raw
|
|
474
|
+
own = {k: v for k, v in _obj(row.get("ext")).items() if k != KEPT}
|
|
475
|
+
if own:
|
|
476
|
+
e["ext"] = copy.deepcopy(own)
|
|
477
|
+
for name, value in _obj(kept.get("eval")).items():
|
|
478
|
+
e.setdefault(name, copy.deepcopy(value))
|
|
479
|
+
return e, observed
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _lay(record: dict, kept: dict, e: dict, skip: Iterable[str]) -> dict:
|
|
483
|
+
for name, value in kept.items():
|
|
484
|
+
if name not in skip and name not in ("ext", "eval"):
|
|
485
|
+
record.setdefault(name, copy.deepcopy(value))
|
|
486
|
+
record.setdefault("tier", "reported")
|
|
487
|
+
record["ext"] = {**copy.deepcopy(_obj(kept.get("ext"))), NS: e}
|
|
488
|
+
return record
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def signal(row: dict, source_kind: str = "judge") -> dict:
|
|
492
|
+
"""The OCP score signal of one reading (a normalised row of `scores.jsonl`)."""
|
|
493
|
+
kept = _obj(_obj(row.get("ext")).get(KEPT))
|
|
494
|
+
e, observed = _eval_ext(row, READING_MEMBERS, kept)
|
|
495
|
+
value = row["value"]
|
|
496
|
+
if isinstance(value, str):
|
|
497
|
+
e["level"], value = value, None
|
|
498
|
+
given = kept.get("id") # the id another tool gave the signal in its run document
|
|
499
|
+
out = {"id": given if isinstance(given, str) and given else row["id"], "kind": "score", "name": row["component"],
|
|
500
|
+
"value": value, "observed_at": observed,
|
|
501
|
+
"source": {**copy.deepcopy(_obj(kept.get("source")) or {"kind": source_kind}), "ref": row["judge"]}}
|
|
502
|
+
if out["id"] != row["id"]:
|
|
503
|
+
e["id"] = row["id"]
|
|
504
|
+
return _lay(out, kept, e, ("source", "id"))
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def preference(row: dict, slate: str, judge_kind: str = "model") -> dict:
|
|
508
|
+
"""The OCP preference of one pair (a normalised row of `pairs.jsonl`), on the slate `slate`."""
|
|
509
|
+
kept = _obj(_obj(row.get("ext")).get(KEPT))
|
|
510
|
+
e, observed = _eval_ext(row, PAIR_MEMBERS, kept)
|
|
511
|
+
judge = copy.deepcopy(_obj(kept.get("judge")) or {"kind": judge_kind})
|
|
512
|
+
judge["ext"] = {**_obj(judge.get("ext")), NS: {"id": row["judge"]}}
|
|
513
|
+
out = {"id": row["id"], "slate": kept.get("slate") or slate,
|
|
514
|
+
"winner": "tie" if row["verdict"] == "tie" else row[row["verdict"]], "members": [row["a"], row["b"]],
|
|
515
|
+
"judge": judge, "observed_at": observed}
|
|
516
|
+
return _lay(out, kept, e, ("judge", "slate"))
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def export(runs: list[dict], scores: list[dict], pairs: list[dict], *, only: Iterable[str] | None = None,
|
|
520
|
+
persons: Iterable[str] = ()) -> tuple[list[dict], list[dict]]:
|
|
521
|
+
"""`(entries, not_used)`: per run, what `eval scores export` prints, and the rows it could not place.
|
|
522
|
+
|
|
523
|
+
An entry is `{"run", "signals", "preferences"}`, with `slate` when the run has preferences. Every
|
|
524
|
+
reading and pair is written, corrected ones too, so nothing is lost on the way back. A preference is
|
|
525
|
+
copied into the entry of both of its runs. Readings and pairs by a judge id in `persons` are written
|
|
526
|
+
as a user's (`source.kind` `user`, `judge.kind` `user`); the others as a judge's and a model's.
|
|
527
|
+
A reading or pair whose run is not in `runs` is listed in `not_used`."""
|
|
528
|
+
persons = set(persons)
|
|
529
|
+
by_id = {row["run"]: row for row in runs}
|
|
530
|
+
wanted = list(by_id) if only is None else list(dict.fromkeys(only))
|
|
531
|
+
entries = {name: {"run": name, "signals": [], "preferences": []} for name in wanted if name in by_id}
|
|
532
|
+
used: dict[str, list[str]] = {}
|
|
533
|
+
not_used = []
|
|
534
|
+
for row in scores:
|
|
535
|
+
if row["run"] not in by_id:
|
|
536
|
+
not_used.append({"kind": "reading", "id": row["id"], "reason": f'no run "{row["run"]}" in the study'})
|
|
537
|
+
elif row["run"] in entries:
|
|
538
|
+
entries[row["run"]]["signals"].append(signal(row, "user" if row["judge"] in persons else "judge"))
|
|
539
|
+
slates: dict[str, dict] = {}
|
|
540
|
+
for row in pairs:
|
|
541
|
+
missing = [name for name in (row["a"], row["b"]) if name not in by_id]
|
|
542
|
+
if missing:
|
|
543
|
+
not_used.append({"kind": "pair", "id": row["id"], "reason": f'no run "{missing[0]}" in the study'})
|
|
544
|
+
continue
|
|
545
|
+
own = [_obj(_obj(_obj(by_id[name].get("ext")).get(KEPT)).get("slate")) for name in (row["a"], row["b"])]
|
|
546
|
+
shared = own[0] if own[0] and own[0].get("id") == own[1].get("id") else None
|
|
547
|
+
record = preference(row, shared["id"] if shared else default_slate(by_id[row["a"]]["input"]),
|
|
548
|
+
"user" if row["judge"] in persons else "model")
|
|
549
|
+
slate = slates.setdefault(record["slate"], copy.deepcopy(shared) if shared and shared["id"] == record["slate"]
|
|
550
|
+
else {"id": record["slate"], "members": []})
|
|
551
|
+
for name in (row["a"], row["b"]):
|
|
552
|
+
if name not in slate["members"]:
|
|
553
|
+
slate["members"].append(name)
|
|
554
|
+
if name in entries:
|
|
555
|
+
entries[name]["preferences"].append(record)
|
|
556
|
+
if record["slate"] not in used.setdefault(name, []):
|
|
557
|
+
used[name].append(record["slate"])
|
|
558
|
+
out = []
|
|
559
|
+
for name in wanted:
|
|
560
|
+
if name not in entries:
|
|
561
|
+
not_used.append({"kind": "run", "id": name, "reason": "no such run in the study"})
|
|
562
|
+
continue
|
|
563
|
+
entry = entries[name]
|
|
564
|
+
names = used.get(name, [])
|
|
565
|
+
if names:
|
|
566
|
+
entry["slate"] = slates[names[0]]
|
|
567
|
+
if len(names) > 1:
|
|
568
|
+
entry["other_slates"] = [slates[other] for other in names[1:]]
|
|
569
|
+
out.append(entry)
|
|
570
|
+
return out, not_used
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def merge(doc: dict, entry: dict) -> dict:
|
|
574
|
+
"""A copy of the OCP run document `doc` with one export entry merged in: the signals and preferences
|
|
575
|
+
that are not there yet, by id, and the slate when the run has none."""
|
|
576
|
+
out = copy.deepcopy(doc)
|
|
577
|
+
run = out["run"]
|
|
578
|
+
if entry.get("run") != run.get("id"):
|
|
579
|
+
raise ValueError(f"the entry is for run {entry.get('run')!r}, the document is run {run.get('id')!r}")
|
|
580
|
+
for name in ("signals", "preferences"):
|
|
581
|
+
have = {record.get("id") for record in run.get(name) or [] if isinstance(record, dict)}
|
|
582
|
+
new = [copy.deepcopy(record) for record in entry.get(name) or [] if record["id"] not in have]
|
|
583
|
+
if new:
|
|
584
|
+
run[name] = list(run.get(name) or []) + new
|
|
585
|
+
if entry.get("slate") is not None and "slate" not in run:
|
|
586
|
+
run["slate"] = copy.deepcopy(entry["slate"])
|
|
587
|
+
return out
|