loopmath 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. loopmath/__init__.py +1 -1
  2. loopmath/cli_registry.py +1 -0
  3. loopmath/eval/__init__.py +6 -0
  4. loopmath/eval/adapters/__init__.py +1 -0
  5. loopmath/eval/adapters/ocp.py +587 -0
  6. loopmath/eval/commands.py +237 -0
  7. loopmath/eval/contracts.py +1212 -0
  8. loopmath/eval/designcheck.py +503 -0
  9. loopmath/eval/engine_pymc.py +364 -0
  10. loopmath/eval/fit.py +279 -0
  11. loopmath/eval/fitfile.py +911 -0
  12. loopmath/eval/judges.py +997 -0
  13. loopmath/eval/labels.py +1268 -0
  14. loopmath/eval/model.py +700 -0
  15. loopmath/eval/quality.py +276 -0
  16. loopmath/eval/rubric.py +552 -0
  17. loopmath/eval/scores.py +902 -0
  18. loopmath/eval/simulate.py +1150 -0
  19. loopmath/eval/specs/__init__.py +38 -0
  20. loopmath/eval/specs/catalog.py +397 -0
  21. loopmath/eval/steps.py +334 -0
  22. loopmath/eval/testing.py +268 -0
  23. loopmath/eval/value.py +418 -0
  24. loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml +1 -1
  25. loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json +1 -1
  26. loopmath/prices.toml +10 -3
  27. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/METADATA +19 -2
  28. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/RECORD +35 -14
  29. {loopmath-0.4.2.data → loopmath-0.4.4.data}/data/spec/ocp-v0.2.schema.json +0 -0
  30. {loopmath-0.4.2.data → loopmath-0.4.4.data}/data/spec/ocp-v0.schema.json +0 -0
  31. {loopmath-0.4.2.data → loopmath-0.4.4.data}/data/spec/ocp_conformance.py +0 -0
  32. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/WHEEL +0 -0
  33. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/entry_points.txt +0 -0
  34. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/licenses/LICENSE +0 -0
  35. {loopmath-0.4.2.dist-info → loopmath-0.4.4.dist-info}/top_level.txt +0 -0
loopmath/__init__.py CHANGED
@@ -5,4 +5,4 @@ The command line (`loopmath`, or its alias `loop`) is the interface; see the
5
5
  README and `loopmath --help`.
6
6
  """
7
7
 
8
- __version__ = "0.4.2"
8
+ __version__ = "0.4.4"
loopmath/cli_registry.py CHANGED
@@ -1248,6 +1248,7 @@ def register(sub) -> None:
1248
1248
  _add_tasks(sub)
1249
1249
  _add_harness(sub)
1250
1250
  _add_flow(sub)
1251
+ importlib.import_module("loopmath.eval.commands").add(sub) # `eval` (0.4.4, experimental)
1251
1252
 
1252
1253
 
1253
1254
  # Handler targets, for the registry test: every one must import and be callable.
@@ -0,0 +1,6 @@
1
+ """`loopmath eval` (experimental): rubric scores from judges and reviewers, read as noisy measurements.
2
+
3
+ Each command is one stage: named files in, one file out, a versioned schema between stages. The package
4
+ imports nothing from `store`, `belief`, `flows`, `compare`, `recommend` or `gate`; only `adapters/ocp.py`
5
+ reads OCP documents, and only `engine_pymc.py` imports PyMC. `tests/eval/test_eval_import_rule.py` checks it.
6
+ """
@@ -0,0 +1 @@
1
+ """Adapters between the score tables of `loopmath eval` and other loopmath documents."""
@@ -0,0 +1,587 @@
1
+ """OCP signals, preferences and harness-runs rows to and from the score tables.
2
+
3
+ Not a command: `scores.py` calls it. The only module of `loopmath.eval` that may import `loopmath.ocp` and
4
+ `loopmath.harness_runs`.
5
+
6
+ The mapping, both ways:
7
+
8
+ - A reading is a score signal. `id` is the reading id, `name` the component, `source.ref` the judge, and
9
+ `ext["dev.loopmath.eval"]` holds `call`, `step`, `rubric`, `replaces`, `assignment`, `item` and `of`.
10
+ A signal's value is a number, so a level that is a word is written as `level` beside them, with the
11
+ value null. `cost_usd` and the members of the reading's own `ext` are kept there too.
12
+ - A pairwise judgement is a preference on a slate of runs of one input: `winner` is a run id or `tie`,
13
+ `members` is `[a, b]`, the judge id sits in `judge.ext["dev.loopmath.eval"].id`, and
14
+ `ext["dev.loopmath.eval"]` holds `component`, `step`, `call`, `shown_first`, `rubric`, `replaces` and
15
+ `assignment`.
16
+ - What an OCP record holds beyond these columns (a tier, a unit, an attempt, another namespace) is kept on
17
+ the study row under `ext["dev.loopmath.eval.ocp"]` and written back on export.
18
+
19
+ Records made by other tools have no `dev.loopmath.eval` member. For those, per (run, judge, name) the
20
+ newest score signal is the reading and each earlier one is a reading it corrects. A signal id of another
21
+ tool is unique in its run document, not in a study, so the reading's id and call are `<run id>.<signal
22
+ id>`: two documents that both hold a signal `s1` give two readings. The signal's own id is kept and
23
+ written back on export, where the eval member's `id` names the reading. A preference is read on the
24
+ overall component, or the one the caller names, with `shown_first` unknown.
25
+
26
+ Nothing here writes a file. `read_documents` and `read_rows` return rows for `scores.py` to validate and
27
+ merge; `export` returns what `eval scores export` prints.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import argparse
33
+ import copy
34
+ import datetime as _dt
35
+ import math
36
+ import re
37
+ from dataclasses import dataclass, field
38
+ from pathlib import Path
39
+ from typing import Any, Iterable, Mapping
40
+
41
+ from ...harness_runs.rows import read_numbered, run_id_for, stage_piece
42
+ from ...ocp.conformance import SIMULATE_KEY, validate_doc
43
+ from .. import contracts as C
44
+
45
+ NS = "dev.loopmath.eval"
46
+ KEPT = "dev.loopmath.eval.ocp"
47
+ READING_MEMBERS = ("call", "step", "rubric", "replaces", "assignment", "item", "of")
48
+ PAIR_MEMBERS = ("component", "step", "call", "shown_first", "rubric", "replaces", "assignment")
49
+ ALSO = ("level", "cost_usd", "observed_at", "ext") # written beside them when the row needs it
50
+ JUDGE_SOURCES = ("judge", "user")
51
+ _OCP_TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$")
52
+ _WORD = re.compile(r"[A-Za-z]+|\d+")
53
+
54
+
55
+ @dataclass
56
+ class Batch:
57
+ """What one source gave: raw rows for the study tables, and every record that was not used.
58
+
59
+ `runs`, `scores` and `pairs` are not validated yet. `source` names where a row came from, by table and
60
+ id, for refusal messages. Each entry of `not_used` is `{"source", "kind", "id", "reason"}`."""
61
+
62
+ runs: list[dict] = field(default_factory=list)
63
+ scores: list[dict] = field(default_factory=list)
64
+ pairs: list[dict] = field(default_factory=list)
65
+ not_used: list[dict] = field(default_factory=list)
66
+ source: dict[tuple[str, str], str] = field(default_factory=dict)
67
+
68
+
69
+ def run(args: argparse.Namespace) -> int:
70
+ """The adapter is not a command. `eval scores` is the command that uses it."""
71
+ from ..scores import run as scores_run
72
+
73
+ return scores_run(args)
74
+
75
+
76
+ def family_of(model_id: str) -> str:
77
+ """A family label for a model id that states none: its first word, lower case (`model-large` gives `model`).
78
+
79
+ A guess, used only when a record names no family. State `model.family` in the record to set it."""
80
+ words = _WORD.findall(model_id)
81
+ return words[0].lower() if words else "unknown"
82
+
83
+
84
+ def default_slate(input_id: str) -> str:
85
+ """The slate id `export` gives the pairs of one input that came with no slate of their own."""
86
+ return f"eval.{input_id}"[:200]
87
+
88
+
89
+ def _obj(value: Any) -> dict:
90
+ return value if isinstance(value, dict) else {}
91
+
92
+
93
+ def _instant(value: Any) -> float:
94
+ try:
95
+ return _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
96
+ except (ValueError, OverflowError, OSError):
97
+ return 0.0
98
+
99
+
100
+ def _total(costs: Iterable[Any]) -> float | None:
101
+ """The sum of the costs that are numbers, or None when there is none or the sum is not a finite number."""
102
+ numbers = [x for x in costs if isinstance(x, (int, float)) and not isinstance(x, bool)]
103
+ try:
104
+ total = float(sum(numbers))
105
+ except OverflowError:
106
+ return None
107
+ return total if numbers and math.isfinite(total) else None
108
+
109
+
110
+ def _ident(text: str) -> str:
111
+ return "".join("_" if ch.isspace() or ch in "/@" else ch for ch in text)
112
+
113
+
114
+ # ---------------------------------------------------------------------------------------------------
115
+ # OCP run documents to rows
116
+
117
+ def _step(step: str, model: Any, effort: Any, ext: Any = None) -> dict:
118
+ ref = model if isinstance(model, dict) else {"id": model}
119
+ model_id = ref.get("id") or ref.get("raw") or "unknown"
120
+ family = ref.get("family") if isinstance(ref.get("family"), str) and ref.get("family") else family_of(model_id)
121
+ out: dict = {"step": step, "model": {"id": model_id, "family": _ident(family)}}
122
+ if isinstance(effort, str) and effort:
123
+ out["effort"] = effort
124
+ variant = _obj(_obj(ext).get(NS)).get("variant")
125
+ if isinstance(variant, str) and variant:
126
+ out["variant"] = variant
127
+ return out
128
+
129
+
130
+ def run_row(doc: dict, file: str) -> dict:
131
+ """The study's run row of one OCP run document.
132
+
133
+ `input` is `run.task.id`. `arm` is `run.flow.arm`, else `run.labels.arm`, else the configuration id.
134
+ `steps` are the workflow's pieces in order with their settings; without a configuration, the last
135
+ attempt of each node. `cost_usd` is the sum of the attempts' `cost.usd`."""
136
+ run = _obj(doc.get("run"))
137
+ task, flow, config = _obj(run.get("task")), _obj(run.get("flow")), _obj(run.get("configuration"))
138
+ if not isinstance(run.get("id"), str) or not isinstance(task.get("id"), str):
139
+ raise C.ContractError(file, "run", "the document has no run.id or no run.task.id",
140
+ "give an OCP run document at version 0.3 or later, with its task", "no_run")
141
+ arm = flow.get("arm") or _obj(run.get("labels")).get("arm") or config.get("id")
142
+ if not isinstance(arm, str) or not arm:
143
+ raise C.ContractError(file, "run", f'run "{run["id"]}" names no arm',
144
+ "state run.flow.arm, run.labels.arm or a configuration id", "no_arm")
145
+ out: dict = {"run": run["id"], "input": task["id"], "arm": arm}
146
+ if isinstance(flow.get("split"), str) and flow["split"]:
147
+ out["split"] = flow["split"]
148
+ settings = _obj(config.get("settings"))
149
+ pieces = [p.get("id") for p in _obj(config.get("workflow")).get("pieces") or [] if isinstance(p, dict)]
150
+ order = [p for p in pieces if p in settings] + [p for p in settings if p not in pieces]
151
+ steps = [_step(_ident(p), _obj(settings[p]).get("model") or {}, _obj(settings[p]).get("effort"),
152
+ _obj(settings[p]).get("ext")) for p in order if isinstance(settings[p], dict)]
153
+ attempts = [a for a in doc.get("attempts") or [] if isinstance(a, dict)]
154
+ if not steps:
155
+ last: dict[str, dict] = {}
156
+ for attempt in attempts:
157
+ where = attempt.get("vertex") or attempt.get("node")
158
+ if isinstance(where, str) and where:
159
+ last.pop(where, None)
160
+ last[where] = attempt
161
+ steps = [_step(_ident(where), a.get("model") or {}, a.get("effort"), a.get("ext")) for where, a in last.items()]
162
+ if not steps:
163
+ raise C.ContractError(file, "run", f'run "{run["id"]}" has no step settings',
164
+ "give run.configuration.settings, or attempts with their models", "no_steps")
165
+ out["steps"] = steps
166
+ cost = _total(_obj(a.get("cost")).get("usd") for a in attempts)
167
+ if cost is not None:
168
+ out["cost_usd"] = cost
169
+ out["simulated"] = isinstance(_obj(run.get("ext")).get(SIMULATE_KEY), dict)
170
+ slate = run.get("slate")
171
+ if isinstance(slate, dict) and (slate.get("id") != default_slate(task["id"]) or set(slate) - {"id", "members"}):
172
+ out["ext"] = {KEPT: {"slate": copy.deepcopy(slate)}} # a slate `export` would not write by itself
173
+ return out
174
+
175
+
176
+ def _rest(record: dict, columns: Iterable[str], known: Iterable[str]) -> dict:
177
+ """What an OCP record holds beyond the study row's columns, for `ext[KEPT]`."""
178
+ rest = {k: copy.deepcopy(v) for k, v in record.items() if k not in columns and k != "ext"}
179
+ if rest.get("tier") == "reported":
180
+ del rest["tier"]
181
+ ext = {k: copy.deepcopy(v) for k, v in _obj(record.get("ext")).items() if k != NS}
182
+ if ext:
183
+ rest["ext"] = ext
184
+ more = {k: copy.deepcopy(v) for k, v in _obj(_obj(record.get("ext")).get(NS)).items()
185
+ if k not in known and k not in ALSO}
186
+ if more:
187
+ rest["eval"] = more
188
+ return rest
189
+
190
+
191
+ def _common(row: dict, e: dict, names: Iterable[str], rest: dict) -> dict:
192
+ for name in names:
193
+ if e.get(name) is not None:
194
+ row[name] = e[name]
195
+ ext = dict(_obj(e.get("ext")))
196
+ if rest:
197
+ ext[KEPT] = rest
198
+ if ext:
199
+ row["ext"] = ext
200
+ return row
201
+
202
+
203
+ def scoped_id(run_id: str, record_id: str) -> str:
204
+ """The study-wide id of a record that another tool named within its run document."""
205
+ return _ident(f"{run_id}.{record_id}")
206
+
207
+
208
+ def reading_row(run_id: str, signal: dict, rubric_ref: str | None = None, *, scope: bool = False) -> dict:
209
+ """The study's reading of one score signal. Not validated: `contracts.validate_score` does that.
210
+
211
+ The reading's id is the eval member's `id`, else the signal's id. With `scope`, a signal that has no
212
+ eval member is named within its run: its reading id and call are `scoped_id(run, signal id)`. When
213
+ the reading's id is not the signal's, the signal's id is kept for `signal` to write back."""
214
+ stated = _obj(signal.get("ext")).get(NS)
215
+ e = _obj(stated)
216
+ source = _obj(signal.get("source"))
217
+ given = signal.get("id")
218
+ ident = e.get("id") or given
219
+ if scope and not isinstance(stated, dict) and isinstance(given, str) and given:
220
+ ident = scoped_id(run_id, given)
221
+ row = {"id": ident, "run": run_id, "step": e.get("step") or C.FINAL, "component": signal.get("name"),
222
+ "judge": source.get("ref"), "call": e.get("call") or ident,
223
+ "value": e["level"] if e.get("level") is not None else signal.get("value"),
224
+ "rubric": e.get("rubric") or rubric_ref, "observed_at": e.get("observed_at") or signal.get("observed_at")}
225
+ rest = _rest(signal, ("id", "kind", "name", "value", "observed_at", "source"), (*READING_MEMBERS, "id"))
226
+ if ident != given:
227
+ rest["id"] = given
228
+ kept_source = {k: copy.deepcopy(v) for k, v in source.items() if k != "ref"}
229
+ if kept_source not in ({"kind": "judge"}, {"kind": "user"}): # which of the two: the judges file says
230
+ rest["source"] = kept_source
231
+ return _common(row, e, ("item", "of", "replaces", "assignment", "cost_usd"), rest)
232
+
233
+
234
+ def _judge_id(judge: dict) -> str:
235
+ stated = _obj(_obj(judge.get("ext")).get(NS)).get("id")
236
+ if isinstance(stated, str) and stated:
237
+ return stated
238
+ return _ident(str(judge.get("model") or judge.get("kind") or "referee"))
239
+
240
+
241
+ def pair_row(preference: dict, members: list, inputs: Mapping[str, str], rubric_ref: str | None = None,
242
+ component: str | None = None, observed_at: str | None = None) -> dict:
243
+ """The study's pair of one preference between the two runs `members`. Not validated."""
244
+ e = _obj(_obj(preference.get("ext")).get(NS))
245
+ a, b = members
246
+ judge = _obj(preference.get("judge"))
247
+ winner = preference.get("winner")
248
+ row = {"id": preference.get("id"), "a": a, "b": b, "step": e.get("step") or C.FINAL,
249
+ "component": e.get("component") or component, "judge": _judge_id(judge),
250
+ "call": e.get("call") or preference.get("id"), "shown_first": e.get("shown_first"),
251
+ "verdict": "a" if winner == a else "b" if winner == b else winner,
252
+ "rubric": e.get("rubric") or rubric_ref,
253
+ "observed_at": e.get("observed_at") or preference.get("observed_at") or observed_at}
254
+ rest = _rest(preference, ("id", "winner", "members", "observed_at", "judge", "slate"), PAIR_MEMBERS)
255
+ kept_judge = {k: copy.deepcopy(v) for k, v in judge.items() if k != "ext"}
256
+ judge_ext = {k: copy.deepcopy(v) for k, v in _obj(judge.get("ext")).items() if k != NS}
257
+ if judge_ext:
258
+ kept_judge["ext"] = judge_ext
259
+ if kept_judge and kept_judge not in ({"kind": "model"}, {"kind": "user"}):
260
+ rest["judge"] = kept_judge
261
+ if preference.get("slate") != default_slate(inputs.get(a, "")):
262
+ rest["slate"] = preference.get("slate")
263
+ return _common(row, e, ("replaces", "assignment", "cost_usd"), rest)
264
+
265
+
266
+ def _signals(run_id: str, signals: Iterable[Any], where: str, batch: Batch, rubric_ref: str | None,
267
+ components: set[str] | None, *, scope: bool = False) -> None:
268
+ """Add the readings of one run's signals to `batch`. Verdicts and events are not readings and are skipped.
269
+
270
+ `scope` is for run documents, whose signal ids are unique per document only (see `reading_row`)."""
271
+ plain: dict[tuple[str, Any], list[tuple[float, int, dict]]] = {}
272
+
273
+ def skip(signal: dict, reason: str) -> None:
274
+ batch.not_used.append({"source": where, "kind": "signal", "id": signal.get("id"), "reason": reason})
275
+
276
+ for n, signal in enumerate(signals):
277
+ if not isinstance(signal, dict) or signal.get("kind") != "score":
278
+ continue
279
+ source = _obj(signal.get("source"))
280
+ e = _obj(signal.get("ext")).get(NS)
281
+ if source.get("kind") not in JUDGE_SOURCES or not isinstance(source.get("ref"), str) or not source["ref"]:
282
+ skip(signal, "a score with no judge: source.kind is not judge or user, or source.ref is missing")
283
+ elif signal.get("value") is None and _obj(e).get("level") is None:
284
+ skip(signal, "a declared score with no value yet")
285
+ elif isinstance(e, dict):
286
+ row = reading_row(run_id, signal, rubric_ref, scope=scope)
287
+ batch.scores.append(row)
288
+ batch.source[("scores", str(row["id"]))] = where
289
+ elif rubric_ref is None:
290
+ skip(signal, "the signal names no rubric; give --rubric")
291
+ elif components is not None and signal.get("name") not in components:
292
+ skip(signal, f'"{signal.get("name")}" is not a component of the rubric')
293
+ else:
294
+ plain.setdefault((source["ref"], signal.get("name")), []).append(
295
+ (_instant(signal.get("observed_at")), n, signal))
296
+ for group in plain.values():
297
+ earlier = None
298
+ for _, _, signal in sorted(group, key=lambda entry: entry[:2]):
299
+ row = reading_row(run_id, signal, rubric_ref, scope=scope)
300
+ if earlier is not None:
301
+ row["replaces"] = earlier
302
+ earlier = row["id"]
303
+ batch.scores.append(row)
304
+ batch.source[("scores", str(row["id"]))] = where
305
+
306
+
307
+ def read_documents(documents: Iterable[tuple[str, Any]], *, rubric: dict | None = None,
308
+ pair_component: str | None = None, inputs: Mapping[str, str] | None = None) -> Batch:
309
+ """Runs, readings and pairs of OCP run documents, given as `(file name, parsed document)`.
310
+
311
+ A document that does not conform to OCP is a refusal that names the file and the place. `inputs` maps
312
+ the runs already in the study to their inputs, so a preference can name a run imported earlier."""
313
+ batch = Batch()
314
+ rubric_ref = None if rubric is None else C.rubric_ref(rubric)
315
+ components = None if rubric is None else {comp["id"] for comp in rubric["components"]}
316
+ known = dict(inputs or {})
317
+ docs = []
318
+ for file, doc in documents:
319
+ if not isinstance(doc, dict) or doc.get("kind") == "flow" or not isinstance(doc.get("run"), dict):
320
+ raise C.ContractError(file, "", "not an OCP run document", "give run documents, one per file", "no_run")
321
+ for finding in validate_doc(doc):
322
+ if finding.level == "error":
323
+ raise C.ContractError(file, finding.path, f"{finding.code}: {finding.message}",
324
+ "fix the document (loopmath ocp validate names every problem)", "bad_ocp")
325
+ row = run_row(doc, file)
326
+ batch.runs.append(row)
327
+ batch.source[("runs", row["run"])] = file
328
+ known[row["run"]] = row["input"]
329
+ docs.append((file, doc))
330
+ seen: dict[Any, dict] = {}
331
+ for file, doc in docs:
332
+ run = doc["run"]
333
+ _signals(run["id"], run.get("signals") or [], file, batch, rubric_ref, components, scope=True)
334
+ slate = _obj(run.get("slate"))
335
+ fallback_time = run.get("ended_at") or run.get("started_at") or _obj(doc.get("producer")).get("emitted_at")
336
+ for preference in run.get("preferences") or []:
337
+ if not isinstance(preference, dict):
338
+ continue
339
+ if seen.get(preference.get("id")) == preference:
340
+ continue # copied into every member's file: read once
341
+ seen[preference.get("id")] = preference
342
+ reason, row = None, None
343
+ has_eval = isinstance(_obj(preference.get("ext")).get(NS), dict)
344
+ members = preference.get("members")
345
+ if not isinstance(members, list) and preference.get("slate") == slate.get("id"):
346
+ members = slate.get("members")
347
+ if not isinstance(members, list) or len(members) != 2:
348
+ reason = "the preference does not name exactly two runs"
349
+ elif preference.get("winner") not in (*members, "tie"):
350
+ reason = "the winner is neither of the two runs nor tie"
351
+ elif not has_eval and (members[0] not in known or members[1] not in known):
352
+ reason = "the preference does not resolve to two runs of the study"
353
+ elif not has_eval and known[members[0]] != known[members[1]]:
354
+ reason = "the two runs have different inputs"
355
+ elif not has_eval and rubric_ref is None:
356
+ reason = "the preference names no rubric; give --rubric"
357
+ else:
358
+ row = pair_row(preference, members, known, rubric_ref, pair_component, fallback_time)
359
+ if row["observed_at"] is None:
360
+ reason = "the preference has no observed_at and its run has no time"
361
+ if reason is not None or row is None:
362
+ batch.not_used.append({"source": file, "kind": "preference", "id": preference.get("id"),
363
+ "reason": reason})
364
+ continue
365
+ batch.pairs.append(row)
366
+ batch.source[("pairs", str(row["id"]))] = file
367
+ return batch
368
+
369
+
370
+ # ---------------------------------------------------------------------------------------------------
371
+ # harness-runs rows to rows
372
+
373
+ def read_rows(path: Any, *, rubric: dict | None = None) -> Batch:
374
+ """Runs and readings of a harness-runs file. A row that breaks its format is a refusal by line.
375
+
376
+ The run's `input` is `task_id`, its `arm` is `flow.arm` or `arm`, its steps are the stages (the last
377
+ stage of each piece), else one step `final` with the row's model. Each `outcome.judges` item is a
378
+ reading: `ext["dev.loopmath.eval"]` on the item states `component` and the members a signal holds. An item
379
+ without it is read on the rubric's overall component, or its only component. The row's own
380
+ `outcome.score` is a reading when the row names its `judge` and its `name` is a component."""
381
+ file = Path(path).name
382
+ try:
383
+ rows = read_numbered(path)
384
+ except ValueError as exc:
385
+ where, problem = "", str(exc)
386
+ if problem.startswith(f"{path}:"): # "<path>:<line>: <problem>"
387
+ line, _, rest = problem[len(f"{path}:"):].partition(": ")
388
+ where, problem = (f"line {line}", rest) if line.isdigit() and rest else ("", problem)
389
+ raise C.ContractError(file, where, problem, "fix the row, or check the path", "bad_rows") from None
390
+ batch = Batch()
391
+ rubric_ref = None if rubric is None else C.rubric_ref(rubric)
392
+ components = None if rubric is None else {comp["id"] for comp in rubric["components"]}
393
+ default = None
394
+ if rubric is not None:
395
+ default = C.overall_component(rubric) or (rubric["components"][0]["id"] if len(rubric["components"]) == 1
396
+ else None)
397
+ for line, row in rows:
398
+ where = f"{file} line {line}"
399
+ run_id = run_id_for(row)
400
+ flow, outcome = _obj(row.get("flow")), _obj(row.get("outcome"))
401
+ arm = flow.get("arm") or row.get("arm")
402
+ if not isinstance(arm, str) or not arm:
403
+ raise C.ContractError(file, f"line {line}", "the row names no arm", "state arm, or flow.arm", "no_arm")
404
+ steps: dict[str, dict] = {}
405
+ costs = [_obj(row.get("cost")).get("usd")]
406
+ for stage in row.get("stages") or []:
407
+ piece = _ident(stage_piece(stage)[0])
408
+ steps.pop(piece, None)
409
+ steps[piece] = _step(piece, stage.get("model") or row.get("model") or "unknown",
410
+ stage.get("effort") or row.get("effort"), stage.get("ext"))
411
+ costs.append(_obj(stage.get("cost")).get("usd"))
412
+ if not steps:
413
+ steps[C.FINAL] = _step(C.FINAL, row.get("model") or "unknown", row.get("effort"), row.get("ext"))
414
+ out: dict = {"run": run_id, "input": row["task_id"], "arm": arm}
415
+ if isinstance(flow.get("split"), str) and flow["split"]:
416
+ out["split"] = flow["split"]
417
+ out["steps"] = list(steps.values())
418
+ cost = _total(costs)
419
+ if cost is not None:
420
+ out["cost_usd"] = cost
421
+ out["simulated"] = isinstance(_obj(row.get("ext")).get(SIMULATE_KEY), dict)
422
+ batch.runs.append(out)
423
+ batch.source[("runs", run_id)] = where
424
+ at = outcome.get("observed_at") or row.get("ended_at") or row.get("started_at")
425
+ signals = []
426
+ if "score" in outcome and isinstance(outcome.get("judge"), str):
427
+ signals.append({"id": f"{run_id}:score", "kind": "score", "name": outcome.get("name") or "judge",
428
+ "value": outcome["score"], "observed_at": at,
429
+ "source": {"kind": "judge", "ref": outcome["judge"]}})
430
+ judges = outcome.get("judges") if isinstance(outcome.get("judges"), list) else []
431
+ for n, item in enumerate(judges):
432
+ if not isinstance(item, dict):
433
+ continue
434
+ e = _obj(_obj(item.get("ext")).get(NS))
435
+ name = e.get("component") or default
436
+ signal = {"id": e.get("id") or f"{run_id}:judge:{n}", "kind": "score", "name": name,
437
+ "value": item.get("score"), "observed_at": item.get("observed_at") or at,
438
+ "source": {"kind": "judge", "ref": item.get("id")}}
439
+ if e:
440
+ signal["ext"] = {NS: {k: v for k, v in e.items() if k not in ("component", "id")}}
441
+ reason = None
442
+ if name is None:
443
+ reason = ("the judge score names no component; give --rubric" if rubric is None else
444
+ "the judge score names no component, and the rubric has no overall component and "
445
+ "more than one component")
446
+ elif signal["observed_at"] is None and (signal["value"] is not None or e.get("level") is not None):
447
+ reason = "the judge score has no observed_at and its row has no time"
448
+ if reason is not None:
449
+ batch.not_used.append({"source": where, "kind": "signal", "id": signal["id"], "reason": reason})
450
+ continue
451
+ signals.append(signal)
452
+ _signals(run_id, signals, where, batch, rubric_ref, components)
453
+ return batch
454
+
455
+
456
+ # ---------------------------------------------------------------------------------------------------
457
+ # Rows to OCP signals and preferences
458
+
459
+ def _ocp_time(text: str) -> tuple[str, str | None]:
460
+ """`(the time as OCP writes it, the text as given when the two differ)`."""
461
+ if _OCP_TIME.match(text):
462
+ return text, None
463
+ parsed = _dt.datetime.fromisoformat(text[:-1] + "+00:00" if text.endswith("Z") else text)
464
+ return parsed.isoformat(), text
465
+
466
+
467
+ def _eval_ext(row: dict, names: Iterable[str], kept: dict) -> tuple[dict, str]:
468
+ e: dict = {name: row[name] for name in names if row.get(name) is not None}
469
+ if row.get("cost_usd") is not None:
470
+ e["cost_usd"] = row["cost_usd"]
471
+ observed, raw = _ocp_time(row["observed_at"])
472
+ if raw is not None:
473
+ e["observed_at"] = raw
474
+ own = {k: v for k, v in _obj(row.get("ext")).items() if k != KEPT}
475
+ if own:
476
+ e["ext"] = copy.deepcopy(own)
477
+ for name, value in _obj(kept.get("eval")).items():
478
+ e.setdefault(name, copy.deepcopy(value))
479
+ return e, observed
480
+
481
+
482
+ def _lay(record: dict, kept: dict, e: dict, skip: Iterable[str]) -> dict:
483
+ for name, value in kept.items():
484
+ if name not in skip and name not in ("ext", "eval"):
485
+ record.setdefault(name, copy.deepcopy(value))
486
+ record.setdefault("tier", "reported")
487
+ record["ext"] = {**copy.deepcopy(_obj(kept.get("ext"))), NS: e}
488
+ return record
489
+
490
+
491
+ def signal(row: dict, source_kind: str = "judge") -> dict:
492
+ """The OCP score signal of one reading (a normalised row of `scores.jsonl`)."""
493
+ kept = _obj(_obj(row.get("ext")).get(KEPT))
494
+ e, observed = _eval_ext(row, READING_MEMBERS, kept)
495
+ value = row["value"]
496
+ if isinstance(value, str):
497
+ e["level"], value = value, None
498
+ given = kept.get("id") # the id another tool gave the signal in its run document
499
+ out = {"id": given if isinstance(given, str) and given else row["id"], "kind": "score", "name": row["component"],
500
+ "value": value, "observed_at": observed,
501
+ "source": {**copy.deepcopy(_obj(kept.get("source")) or {"kind": source_kind}), "ref": row["judge"]}}
502
+ if out["id"] != row["id"]:
503
+ e["id"] = row["id"]
504
+ return _lay(out, kept, e, ("source", "id"))
505
+
506
+
507
+ def preference(row: dict, slate: str, judge_kind: str = "model") -> dict:
508
+ """The OCP preference of one pair (a normalised row of `pairs.jsonl`), on the slate `slate`."""
509
+ kept = _obj(_obj(row.get("ext")).get(KEPT))
510
+ e, observed = _eval_ext(row, PAIR_MEMBERS, kept)
511
+ judge = copy.deepcopy(_obj(kept.get("judge")) or {"kind": judge_kind})
512
+ judge["ext"] = {**_obj(judge.get("ext")), NS: {"id": row["judge"]}}
513
+ out = {"id": row["id"], "slate": kept.get("slate") or slate,
514
+ "winner": "tie" if row["verdict"] == "tie" else row[row["verdict"]], "members": [row["a"], row["b"]],
515
+ "judge": judge, "observed_at": observed}
516
+ return _lay(out, kept, e, ("judge", "slate"))
517
+
518
+
519
+ def export(runs: list[dict], scores: list[dict], pairs: list[dict], *, only: Iterable[str] | None = None,
520
+ persons: Iterable[str] = ()) -> tuple[list[dict], list[dict]]:
521
+ """`(entries, not_used)`: per run, what `eval scores export` prints, and the rows it could not place.
522
+
523
+ An entry is `{"run", "signals", "preferences"}`, with `slate` when the run has preferences. Every
524
+ reading and pair is written, corrected ones too, so nothing is lost on the way back. A preference is
525
+ copied into the entry of both of its runs. Readings and pairs by a judge id in `persons` are written
526
+ as a user's (`source.kind` `user`, `judge.kind` `user`); the others as a judge's and a model's.
527
+ A reading or pair whose run is not in `runs` is listed in `not_used`."""
528
+ persons = set(persons)
529
+ by_id = {row["run"]: row for row in runs}
530
+ wanted = list(by_id) if only is None else list(dict.fromkeys(only))
531
+ entries = {name: {"run": name, "signals": [], "preferences": []} for name in wanted if name in by_id}
532
+ used: dict[str, list[str]] = {}
533
+ not_used = []
534
+ for row in scores:
535
+ if row["run"] not in by_id:
536
+ not_used.append({"kind": "reading", "id": row["id"], "reason": f'no run "{row["run"]}" in the study'})
537
+ elif row["run"] in entries:
538
+ entries[row["run"]]["signals"].append(signal(row, "user" if row["judge"] in persons else "judge"))
539
+ slates: dict[str, dict] = {}
540
+ for row in pairs:
541
+ missing = [name for name in (row["a"], row["b"]) if name not in by_id]
542
+ if missing:
543
+ not_used.append({"kind": "pair", "id": row["id"], "reason": f'no run "{missing[0]}" in the study'})
544
+ continue
545
+ own = [_obj(_obj(_obj(by_id[name].get("ext")).get(KEPT)).get("slate")) for name in (row["a"], row["b"])]
546
+ shared = own[0] if own[0] and own[0].get("id") == own[1].get("id") else None
547
+ record = preference(row, shared["id"] if shared else default_slate(by_id[row["a"]]["input"]),
548
+ "user" if row["judge"] in persons else "model")
549
+ slate = slates.setdefault(record["slate"], copy.deepcopy(shared) if shared and shared["id"] == record["slate"]
550
+ else {"id": record["slate"], "members": []})
551
+ for name in (row["a"], row["b"]):
552
+ if name not in slate["members"]:
553
+ slate["members"].append(name)
554
+ if name in entries:
555
+ entries[name]["preferences"].append(record)
556
+ if record["slate"] not in used.setdefault(name, []):
557
+ used[name].append(record["slate"])
558
+ out = []
559
+ for name in wanted:
560
+ if name not in entries:
561
+ not_used.append({"kind": "run", "id": name, "reason": "no such run in the study"})
562
+ continue
563
+ entry = entries[name]
564
+ names = used.get(name, [])
565
+ if names:
566
+ entry["slate"] = slates[names[0]]
567
+ if len(names) > 1:
568
+ entry["other_slates"] = [slates[other] for other in names[1:]]
569
+ out.append(entry)
570
+ return out, not_used
571
+
572
+
573
+ def merge(doc: dict, entry: dict) -> dict:
574
+ """A copy of the OCP run document `doc` with one export entry merged in: the signals and preferences
575
+ that are not there yet, by id, and the slate when the run has none."""
576
+ out = copy.deepcopy(doc)
577
+ run = out["run"]
578
+ if entry.get("run") != run.get("id"):
579
+ raise ValueError(f"the entry is for run {entry.get('run')!r}, the document is run {run.get('id')!r}")
580
+ for name in ("signals", "preferences"):
581
+ have = {record.get("id") for record in run.get(name) or [] if isinstance(record, dict)}
582
+ new = [copy.deepcopy(record) for record in entry.get(name) or [] if record["id"] not in have]
583
+ if new:
584
+ run[name] = list(run.get(name) or []) + new
585
+ if entry.get("slate") is not None and "slate" not in run:
586
+ run["slate"] = copy.deepcopy(entry["slate"])
587
+ return out