nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
"""Judgement: asking the judge about a trial, a job's trials, or matched
|
|
2
|
+
pairs; what the pages show comes from the workspace store. Advisory beside
|
|
3
|
+
the recorded evidence, never in its place."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import threading
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
9
|
+
from flask import abort, jsonify, redirect, request, url_for
|
|
10
|
+
from nullius.judge import batch, client, lanes, questions as Q, state as judge_state, store
|
|
11
|
+
from nullius.runs import feasibility
|
|
12
|
+
from nullius.traces import trajectory
|
|
13
|
+
|
|
14
|
+
from .core import app
|
|
15
|
+
from . import shared
|
|
16
|
+
from . import settings as settings_web
|
|
17
|
+
from . import traces
|
|
18
|
+
|
|
19
|
+
def key(lane: str = "system-one") -> client.Judge:
|
|
20
|
+
"""A judgement's judge, with its key and model: true when it can be
|
|
21
|
+
asked. System One unless the LLM judgement is named; passed down to
|
|
22
|
+
every call as the key is."""
|
|
23
|
+
return client.judge_for(lane, os.environ)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Judge calls in flight at once for a batch (Judge all, a comparison's sides, task instructions). One
|
|
27
|
+
# at a time, a 500-trial job at 2-5 s a call took most of an hour inside the click's request.
|
|
28
|
+
CONCURRENCY = 8
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def run_batch(items, ask, name) -> tuple:
|
|
32
|
+
"""``ask(item)`` for every item, ``CONCURRENCY`` at a time; (how many were
|
|
33
|
+
judged, the failures named by ``name(item)``), in the items' order. A
|
|
34
|
+
refused key or a judge that cannot be asked (status 401, 0), a spent
|
|
35
|
+
credit (402), or a judge still busy after its retries (429, 503, 529)
|
|
36
|
+
starts no further call: every other one would fail the same way, and a
|
|
37
|
+
batch going on through a rate limit only lengthens it."""
|
|
38
|
+
stop = threading.Event()
|
|
39
|
+
|
|
40
|
+
def one(item):
|
|
41
|
+
if stop.is_set():
|
|
42
|
+
return None
|
|
43
|
+
try:
|
|
44
|
+
ask(item)
|
|
45
|
+
return True
|
|
46
|
+
except client.JudgeError as e:
|
|
47
|
+
if e.status in (401, 0, 402, 429, 503, 529):
|
|
48
|
+
stop.set()
|
|
49
|
+
return f"{name(item)}: {e}"
|
|
50
|
+
done, failed = 0, []
|
|
51
|
+
with ThreadPoolExecutor(CONCURRENCY) as pool:
|
|
52
|
+
for r in pool.map(one, items):
|
|
53
|
+
if r is True:
|
|
54
|
+
done += 1
|
|
55
|
+
elif r:
|
|
56
|
+
failed.append(r)
|
|
57
|
+
return done, failed
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def ws_dir():
|
|
61
|
+
ws = shared.STATE.get("workspace")
|
|
62
|
+
return ws.dir if ws else None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def judgement_of(job, trial, lane: str = "system-one") -> dict | None:
|
|
66
|
+
d = ws_dir()
|
|
67
|
+
return store.load(d, job.job_id, trial.trial_name, lane=lane) if d else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def panel(job, trial, lane: str = "system-one") -> dict:
|
|
71
|
+
"""What the trial page renders: each question with its answer or 'not
|
|
72
|
+
judged', the record's provenance, whether a Judge action is possible.
|
|
73
|
+
A record asked under an earlier vocabulary is shown under its own
|
|
74
|
+
version and marked `earlier`; Judge again asks the current one."""
|
|
75
|
+
rec = judgement_of(job, trial, lane) or {}
|
|
76
|
+
ids = Q.for_trial(trial.outcome == "resolved", trial.outcome)
|
|
77
|
+
answers = rec.get("answers") or {}
|
|
78
|
+
hygiene = rec.get("hygiene") or {}
|
|
79
|
+
rows = [row_of(q, answers.get(q), hygiene.get(Q.state_of(q))) for q in ids]
|
|
80
|
+
flags = [r for r in rows if r["flag"]]
|
|
81
|
+
sections = [("Outcome", [r for r in rows if not r["flag"]]), ("Behaviour flags", flags)]
|
|
82
|
+
return {"rows": rows, "sections": [(name, rs) for name, rs in sections if rs], "record": rec, "judged": bool(answers), "keyed": bool(key(lane)), "lane": lane, "judge": key(lane),
|
|
83
|
+
"cards": [r for r in rows if not r["flag"]], "flags": flags,
|
|
84
|
+
"earlier": bool(answers) and not store.is_current(rec),
|
|
85
|
+
"raised": [r["id"] for r in flags if r["raised"] and not r["undecided"]], "undecided": [r["id"] for r in flags if r["undecided"]],
|
|
86
|
+
"cleared": [r["id"] for r in flags if r["answer"] and not r["raised"] and not r["undecided"]],
|
|
87
|
+
"flags_judged": sum(1 for r in flags if r["answer"]), "floors_measured": bool(Q.floors().get("floors")),
|
|
88
|
+
"hygiene": hygiene_summary(hygiene)}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
FLAG_THRESHOLD = 0.5 # a Noul at or above this reads as raised; a threshold for Nouls only, never carried to a Choice or Score
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def row_of(q: str, a, hygiene: dict | None = None) -> dict:
|
|
95
|
+
"""One question's row: the answer, and the judge's own `confidence`
|
|
96
|
+
(Choice and Score) or Nullius's `decisiveness` (Noul) - never both.
|
|
97
|
+
A behaviour flag (`flag` in the vocabulary) is `raised` when its Noul
|
|
98
|
+
is at or above `FLAG_THRESHOLD`. `hygiene` is the reading over the
|
|
99
|
+
state this question read; `warnings` names the hygiene flags raised on
|
|
100
|
+
it, shown beside the answer and changing nothing about it."""
|
|
101
|
+
kind = Q.QUESTIONS[q]["type"]
|
|
102
|
+
flag = bool(Q.QUESTIONS[q].get("flag"))
|
|
103
|
+
floor = Q.floor_of(q)
|
|
104
|
+
undecided = is_undecided(kind, a, floor)
|
|
105
|
+
warnings = hygiene_raised(hygiene)
|
|
106
|
+
row = {"floor": floor, "undecided": undecided, "id": q, "type": kind, "instructions": Q.QUESTIONS[q]["instructions"], "answer": a,
|
|
107
|
+
"label": Q.QUESTIONS[q].get("label") or q.replace("_", " "), "blurb": Q.QUESTIONS[q].get("blurb") or "",
|
|
108
|
+
"confidence": confidence_of(kind, a), "decisiveness": decisiveness_of(kind, a),
|
|
109
|
+
"flag": flag, "raised": is_raised(kind, a) if flag else False,
|
|
110
|
+
"state": Q.state_of(q), "warnings": warnings, "tone": tone_of(q, kind, flag, a, undecided, warnings)}
|
|
111
|
+
row.update(headline_of(row))
|
|
112
|
+
return row
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def headline_of(row: dict) -> dict:
|
|
116
|
+
"""What a card or a result cell shows for one answer: the `headline`
|
|
117
|
+
number, its `unit`, the `pill` (Yes / No / Undecided / Raised / Clear /
|
|
118
|
+
the score's level label) and the `measure` beside it (the judge's
|
|
119
|
+
confidence, or Nullius's decisiveness for a yes/no). Nothing here is a
|
|
120
|
+
new number: it is the answer, formatted."""
|
|
121
|
+
a, kind = row["answer"], row["type"]
|
|
122
|
+
if not a:
|
|
123
|
+
return {"headline": None, "unit": "", "pill": "Not judged", "measure_label": "", "measure": None}
|
|
124
|
+
if kind == "noul":
|
|
125
|
+
p = a.get("noul") or 0.0
|
|
126
|
+
pill = "Undecided" if row["undecided"] else (("Raised" if p >= FLAG_THRESHOLD else "Clear") if row["flag"] else ("Yes" if p >= FLAG_THRESHOLD else "No"))
|
|
127
|
+
return {"headline": f"{p * 100:.0f}", "unit": "%", "pill": pill, "measure_label": "Decisiveness", "measure": row["decisiveness"]}
|
|
128
|
+
if kind == "score":
|
|
129
|
+
legend = a.get("legend") or list(Q.QUESTIONS[row["id"]]["criteria"])
|
|
130
|
+
level = min(int((a.get("score") or 0) + 0.5), len(legend) - 1)
|
|
131
|
+
return {"headline": f"{a.get('score') or 0:.1f}", "unit": f"/ {len(legend) - 1}", "pill": legend[level], "measure_label": "Confidence", "measure": row["confidence"]}
|
|
132
|
+
p = (a.get("probabilities") or {}).get(a.get("choice"))
|
|
133
|
+
return {"headline": f"{(p or 0) * 100:.0f}" if p is not None else None, "unit": "%", "pill": a.get("choice") or "", "measure_label": "Confidence", "measure": row["confidence"]}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def tone_of(q: str, kind: str, flag: bool, a, undecided: bool, warnings) -> str:
|
|
137
|
+
"""The row's background: `good` when the answer reads well for the
|
|
138
|
+
agent (a flag clear, a yes on an outcome question, a top score), `bad`
|
|
139
|
+
when it reads badly (a flag raised, a no, a bottom score), `warn` when
|
|
140
|
+
it is undecided within the judge's spread, a middle score, or drawn
|
|
141
|
+
from a state the hygiene reading flagged; `none` with no answer, and for
|
|
142
|
+
a Choice, whose options are not ranked. Advisory colour, nothing else."""
|
|
143
|
+
if not a:
|
|
144
|
+
return "none"
|
|
145
|
+
if undecided:
|
|
146
|
+
return "warn"
|
|
147
|
+
if kind == "noul":
|
|
148
|
+
p = a.get("noul")
|
|
149
|
+
if not isinstance(p, (int, float)):
|
|
150
|
+
return "none"
|
|
151
|
+
yes = p >= FLAG_THRESHOLD
|
|
152
|
+
tone = ("bad" if yes else "good") if flag else ("good" if yes else "bad")
|
|
153
|
+
elif kind == "score":
|
|
154
|
+
legend = a.get("legend") or list(Q.QUESTIONS[q]["criteria"])
|
|
155
|
+
level = min(int((a.get("score") or 0) + 0.5), len(legend) - 1)
|
|
156
|
+
tone = "good" if level == len(legend) - 1 else ("bad" if level == 0 else "warn")
|
|
157
|
+
else:
|
|
158
|
+
tone = "none"
|
|
159
|
+
return "warn" if warnings and tone == "none" else tone
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# --- hygiene: the state is untrusted text, and the judge read it as such -------------
|
|
163
|
+
|
|
164
|
+
HYGIENE_THRESHOLD = 0.5 # a hygiene Noul at or above this is a warning; its own constant, shared with no other question type or flag
|
|
165
|
+
|
|
166
|
+
HYGIENE_LABELS = {"addresses_the_judge": "text addresses the judge", "credential_visible": "a credential may still be visible"}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def hygiene_raised(reading: dict | None) -> list:
|
|
170
|
+
"""The hygiene flags raised on one state kind's reading: [(id, label,
|
|
171
|
+
p, decided)] - `decided` False when p sits within the judge's measured
|
|
172
|
+
spread of the threshold (`floors.json`), so the warning says it is
|
|
173
|
+
undecided rather than certain. Without a measured floor every raised
|
|
174
|
+
flag is decided."""
|
|
175
|
+
out = []
|
|
176
|
+
for q in Q.hygiene_ids():
|
|
177
|
+
p = (reading or {}).get(q)
|
|
178
|
+
if isinstance(p, (int, float)) and p >= HYGIENE_THRESHOLD:
|
|
179
|
+
floor = Q.floor_of(q)
|
|
180
|
+
out.append((q, HYGIENE_LABELS.get(q, q), p, not (isinstance(floor, (int, float)) and abs(p - HYGIENE_THRESHOLD) < floor)))
|
|
181
|
+
return out
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def hygiene_summary(hygiene: dict) -> dict:
|
|
185
|
+
"""For the panel head: which state kinds were read, and on which a
|
|
186
|
+
flag is raised - `{kinds: [...], flagged: {kind: [(id, label, p)]}}`."""
|
|
187
|
+
flagged = {kind: hygiene_raised(r) for kind, r in hygiene.items()}
|
|
188
|
+
return {"kinds": list(hygiene), "flagged": {k: v for k, v in flagged.items() if v}}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def is_undecided(kind: str, a, floor) -> bool:
|
|
192
|
+
"""A yes/no answer whose probability sits within the judge's measured
|
|
193
|
+
run-to-run spread of the 50% line: another ask could land on the other
|
|
194
|
+
side, so it is shown as undecided rather than raised or clear. Without
|
|
195
|
+
a measured floor nothing is undecided - and the row says no floor was
|
|
196
|
+
measured."""
|
|
197
|
+
if not a or kind != "noul" or not isinstance(floor, (int, float)):
|
|
198
|
+
return False
|
|
199
|
+
p = a.get("noul")
|
|
200
|
+
return isinstance(p, (int, float)) and abs(p - FLAG_THRESHOLD) < floor
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def is_raised(kind: str, a) -> bool:
|
|
204
|
+
"""Whether a yes/no answer reads as yes: its probability at or above the threshold."""
|
|
205
|
+
if not a or kind != "noul":
|
|
206
|
+
return False
|
|
207
|
+
p = a.get("noul")
|
|
208
|
+
return isinstance(p, (int, float)) and p >= FLAG_THRESHOLD
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def confidence_of(kind: str, a) -> float | None:
|
|
212
|
+
"""The confidence the judge returned: Choice and Score carry one, a Noul does not."""
|
|
213
|
+
if not a or kind == "noul":
|
|
214
|
+
return None
|
|
215
|
+
return a.get("confidence")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def decisiveness_of(kind: str, a) -> float | None:
|
|
219
|
+
"""How far a Noul's probability sits from 0.5, scaled to 0..1 - derived
|
|
220
|
+
here, not returned by the judge, and not a confidence: 0.9 and 0.1 are
|
|
221
|
+
equally decisive, and a decisive answer can still be wrong."""
|
|
222
|
+
if not a or kind != "noul":
|
|
223
|
+
return None
|
|
224
|
+
p = a.get("noul")
|
|
225
|
+
return abs(p - 0.5) * 2 if isinstance(p, (int, float)) else None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _states_for(trial, traj, ids) -> dict:
|
|
229
|
+
"""The questions grouped by the state they read, each state built once."""
|
|
230
|
+
return batch.states_for(ids, lambda kind: judge_state.build(kind, traj, trial=trial, trial_dir=trial.dir))
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _trial_ids_and_states(trial, traj) -> dict:
|
|
234
|
+
"""Everything one Judge click asks of a trial, grouped by state: the
|
|
235
|
+
trial page's questions."""
|
|
236
|
+
ids = Q.for_trial(trial.outcome == "resolved", trial.outcome)
|
|
237
|
+
return _states_for(trial, traj, ids)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def judge_trial(job, trial, api_key: str, post=None) -> dict:
|
|
241
|
+
"""Ask every question the trial page shows, one call per state, and keep
|
|
242
|
+
the merged record. Returns the record as stored."""
|
|
243
|
+
if not trial.has_trajectory:
|
|
244
|
+
raise client.JudgeError(0, "this trial has no trajectory to judge")
|
|
245
|
+
traj = trajectory.load(trial.trajectory_path)
|
|
246
|
+
if traj.error:
|
|
247
|
+
raise client.JudgeError(0, "the trajectory could not be read: " + traj.error)
|
|
248
|
+
return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _guard():
|
|
252
|
+
shared._require_csrf()
|
|
253
|
+
if not settings_web._local_client():
|
|
254
|
+
abort(403, description="Judging is asked from a browser on the application host only: it is a paid call.")
|
|
255
|
+
if ws_dir() is None:
|
|
256
|
+
abort(400, description="No workspace is open; judgements live in the workspace.")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@app.route("/judge/trial/<job_id>/<trial_name>", methods=["POST"])
|
|
260
|
+
def judge_trial_route(job_id, trial_name):
|
|
261
|
+
"""One click, one trial: the page's questions asked and stored."""
|
|
262
|
+
_guard()
|
|
263
|
+
job, trial = traces._trial_evidence(job_id, trial_name)
|
|
264
|
+
lane = request.form.get("lane") if request.form.get("lane") in client.LANES else "system-one"
|
|
265
|
+
try:
|
|
266
|
+
rec = judge_trial(job, trial, key(lane))
|
|
267
|
+
except client.JudgeError as e:
|
|
268
|
+
if request.accept_mimetypes.best == "application/json":
|
|
269
|
+
return jsonify(ok=False, error=str(e)), 502 if e.status not in (401, 422) else 400
|
|
270
|
+
from flask import flash
|
|
271
|
+
flash(str(e), "error")
|
|
272
|
+
return redirect(url_for("trial", job_id=job_id, trial_name=trial_name))
|
|
273
|
+
if request.accept_mimetypes.best == "application/json":
|
|
274
|
+
return jsonify(ok=True, record=rec)
|
|
275
|
+
tab = "llm-judge" if lane == "llm" else "judgement"
|
|
276
|
+
return redirect(url_for("trial", job_id=job_id, trial_name=trial_name, tab=tab) + "#" + tab)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@app.route("/judge/job/<job_id>", methods=["POST"])
|
|
280
|
+
def judge_job_route(job_id):
|
|
281
|
+
"""Judge the job's unjudged trials with one judge, the form's ``lane``
|
|
282
|
+
(System One unless it names the LLM judge): the click is the request."""
|
|
283
|
+
job = shared._find_job(shared._jobs_for_request(), job_id)
|
|
284
|
+
if job is None:
|
|
285
|
+
abort(404)
|
|
286
|
+
lane = request.form.get("lane") if request.form.get("lane") in client.LANES else "system-one"
|
|
287
|
+
d = ws_dir()
|
|
288
|
+
judged = store.for_job(d, job_id, lane=lane) if d else {}
|
|
289
|
+
todo = [t for t in job.trials if t.has_trajectory and store._SAFE.sub("_", t.trial_name) not in judged]
|
|
290
|
+
_guard()
|
|
291
|
+
k = key(lane)
|
|
292
|
+
done, failed = run_batch(todo, lambda t: judge_trial(job, t, k), lambda t: t.trial_name)
|
|
293
|
+
from flask import flash
|
|
294
|
+
flash(f"{'LLM judge' if lane == 'llm' else 'System One'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
|
|
295
|
+
return redirect(url_for("job", job_id=job_id) + "#judgement")
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def job_summary(job) -> dict:
|
|
299
|
+
"""For the job page: the failure_cause distribution over judged trials,
|
|
300
|
+
how many trials raised each behaviour flag (counted here, over the
|
|
301
|
+
judged trials only), how many are judged at all, and the same reading
|
|
302
|
+
per trial (`by_trial`) so a chip's count is exactly the rows its filter
|
|
303
|
+
shows. `flag_ids` is every flag in the vocabulary's order - a flag's
|
|
304
|
+
number on the page is its place here, raised or not - and `lanes` is
|
|
305
|
+
the board's record -> judge lanes (`nullius.judge.lanes`), whose status
|
|
306
|
+
for each trial is also in `by_trial`."""
|
|
307
|
+
d = ws_dir()
|
|
308
|
+
judged = store.for_job(d, job.job_id) if d else {}
|
|
309
|
+
flag_ids = [q for q in Q.group_ids("trial") if Q.QUESTIONS[q].get("flag")]
|
|
310
|
+
board = lanes.board(job, judged, key=lambda name: store._SAFE.sub("_", name))
|
|
311
|
+
by_trial = {}
|
|
312
|
+
for t in job.trials:
|
|
313
|
+
rec = judged.get(store._SAFE.sub("_", t.trial_name))
|
|
314
|
+
if rec is None:
|
|
315
|
+
continue
|
|
316
|
+
answers = rec.get("answers") or {}
|
|
317
|
+
a = answers.get("failure_cause")
|
|
318
|
+
by_trial[t.trial_name] = {
|
|
319
|
+
"cause": a["choice"] if a and a.get("choice") else None,
|
|
320
|
+
# raised outside the judge's measured spread; an undecided one is not counted
|
|
321
|
+
"flags": [q for q in flag_ids if is_raised(Q.QUESTIONS[q]["type"], answers.get(q)) and not is_undecided(Q.QUESTIONS[q]["type"], answers.get(q), Q.floor_of(q))],
|
|
322
|
+
"status": board["status"].get(t.trial_name) if board else None,
|
|
323
|
+
}
|
|
324
|
+
causes, raised = {}, {}
|
|
325
|
+
for r in by_trial.values():
|
|
326
|
+
if r["cause"]:
|
|
327
|
+
causes[r["cause"]] = causes.get(r["cause"], 0) + 1
|
|
328
|
+
for q in r["flags"]:
|
|
329
|
+
raised[q] = raised.get(q, 0) + 1
|
|
330
|
+
return {"judged": len(judged), "trials": len(job.trials), "unjudged": len([t for t in job.trials if store._SAFE.sub("_", t.trial_name) not in judged]),
|
|
331
|
+
"causes": sorted(causes.items(), key=lambda kv: -kv[1]), "n_cause": sum(causes.values()), "keyed": bool(key()),
|
|
332
|
+
"flags": sorted(raised.items(), key=lambda kv: (-kv[1], kv[0])), "by_trial": by_trial,
|
|
333
|
+
"truncation": store.truncation_report(d, job.job_id) if d and judged else None,
|
|
334
|
+
"unresolved": sum(1 for t in job.trials if t.outcome != "resolved"),
|
|
335
|
+
"flag_ids": flag_ids, "lanes": board, "llm": _llm_count(d, job)}
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _llm_count(d, job) -> dict:
|
|
339
|
+
"""The LLM judge's own count for the job's button: it judges apart from
|
|
340
|
+
System One, so a trial judged by one is still unjudged by the other."""
|
|
341
|
+
judged = store.for_job(d, job.job_id, lane="llm") if d else {}
|
|
342
|
+
return {"unjudged": len([t for t in job.trials if store._SAFE.sub("_", t.trial_name) not in judged]), "keyed": bool(key("llm"))}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# --- the compare page: each side scored alone, the pick made in code ----------------
|
|
346
|
+
|
|
347
|
+
SIDE_QUESTION = "task_satisfaction"
|
|
348
|
+
LEGACY_QUESTION = "better_of_two" # v1 only: a pair state, a pick by the model; shown, never asked
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def pairs_of(view) -> list:
|
|
352
|
+
"""The matched pairs a two-run comparison can judge: (task, trial A, trial B),
|
|
353
|
+
both with a trajectory. Empty for anything but two runs."""
|
|
354
|
+
if len(view.jobs) != 2:
|
|
355
|
+
return []
|
|
356
|
+
out = []
|
|
357
|
+
for row in view.tasks:
|
|
358
|
+
a, b = (list(row.trials) + [None, None])[:2]
|
|
359
|
+
if a and b and a.has_trajectory and b.has_trajectory:
|
|
360
|
+
out.append((row.task, a, b))
|
|
361
|
+
return out
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def side_ids(trial) -> list:
|
|
365
|
+
"""What judging one side asks: the pair group's questions over the trial
|
|
366
|
+
state, and - free on the same call - everything the trial page asks."""
|
|
367
|
+
own = [q for q in Q.group_ids("pair") if Q.state_of(q) == "trial"]
|
|
368
|
+
return own + [q for q in Q.for_trial(trial.outcome == "resolved", trial.outcome) if q not in own]
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def side_of(job, trial) -> dict | None:
|
|
372
|
+
"""One side's score as the compare page shows it, from the trial's own
|
|
373
|
+
current record: the expectation, the level it rounds to, the judge's
|
|
374
|
+
confidence, the provenance. None until scored under the current vocabulary."""
|
|
375
|
+
d = ws_dir()
|
|
376
|
+
rec = store.load(d, job.job_id, trial.trial_name) if d else None
|
|
377
|
+
if not rec or not store.is_current(rec):
|
|
378
|
+
return None
|
|
379
|
+
a = (rec.get("answers") or {}).get(SIDE_QUESTION)
|
|
380
|
+
if not a or not isinstance(a.get("score"), (int, float)):
|
|
381
|
+
return None
|
|
382
|
+
legend = a.get("legend") or list(Q.QUESTIONS[SIDE_QUESTION]["criteria"])
|
|
383
|
+
level = min(int(a["score"] + 0.5), len(legend) - 1)
|
|
384
|
+
return {"score": a["score"], "top": len(legend) - 1, "label": legend[level], "confidence": a.get("confidence"),
|
|
385
|
+
"asked_at": rec.get("asked_at"), "model": rec["judge"]["model"]}
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def pair_records(view) -> dict:
|
|
389
|
+
"""{task: record} for the pairs judged under v1 - the pair state, the
|
|
390
|
+
model's own pick - under job A in the open workspace. Read, never written."""
|
|
391
|
+
d = ws_dir()
|
|
392
|
+
if d is None or len(view.jobs) != 2:
|
|
393
|
+
return {}
|
|
394
|
+
out = {}
|
|
395
|
+
for task, a, b in pairs_of(view):
|
|
396
|
+
rec = store.load(d, view.jobs[0].job_id, store.pair_key(a.trial_name, b.trial_name))
|
|
397
|
+
if rec and (rec.get("answers") or {}).get(LEGACY_QUESTION):
|
|
398
|
+
out[task] = rec
|
|
399
|
+
return out
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def compare_summary(view) -> dict | None:
|
|
403
|
+
"""For the compare page, per matched task: each side's score, the pick
|
|
404
|
+
made in code (`comparison.better_of`) with ties explicit; a v1 pair
|
|
405
|
+
record shown under its version, marked earlier, where no side is scored
|
|
406
|
+
under v2; the judged-pair population; the unjudged count."""
|
|
407
|
+
if len(view.jobs) != 2:
|
|
408
|
+
return None
|
|
409
|
+
from nullius.compare import comparison
|
|
410
|
+
pairs = pairs_of(view)
|
|
411
|
+
legacy = pair_records(view)
|
|
412
|
+
rows, judged, earlier = {}, 0, 0
|
|
413
|
+
for task, a, b in pairs:
|
|
414
|
+
sa, sb = side_of(view.jobs[0], a), side_of(view.jobs[1], b)
|
|
415
|
+
if sa or sb:
|
|
416
|
+
verdict = comparison.better_of(sa and sa["score"], sb and sb["score"])
|
|
417
|
+
rows[task] = {"a": sa, "b": sb, "verdict": verdict, "earlier": False}
|
|
418
|
+
judged += bool(verdict)
|
|
419
|
+
elif task in legacy:
|
|
420
|
+
ans = legacy[task]["answers"][LEGACY_QUESTION]
|
|
421
|
+
rows[task] = {"a": None, "b": None, "verdict": None, "earlier": True, "choice": ans["choice"], "p": (ans.get("probabilities") or {}).get(ans["choice"]),
|
|
422
|
+
"probabilities": ans.get("probabilities") or {}, "questions": legacy[task].get("questions"), "asked_at": legacy[task].get("asked_at"), "model": legacy[task]["judge"]["model"]}
|
|
423
|
+
earlier += 1
|
|
424
|
+
return {"rows": rows, "pairs": len(pairs), "judged": judged, "earlier": earlier, "unjudged": len(pairs) - judged, "keyed": bool(key()),
|
|
425
|
+
"margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION}
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def judge_side(job, trial, api_key: str, post=None) -> dict | None:
|
|
429
|
+
"""Score one side: the questions it still lacks under the current
|
|
430
|
+
vocabulary, one call per state, merged over the trial's own record -
|
|
431
|
+
the same record the trial page shows. Nothing when it has them all."""
|
|
432
|
+
if not trial.has_trajectory:
|
|
433
|
+
raise client.JudgeError(0, "this trial has no trajectory to judge")
|
|
434
|
+
d = ws_dir()
|
|
435
|
+
rec = store.load(d, job.job_id, trial.trial_name)
|
|
436
|
+
have = (rec.get("answers") or {}) if store.is_current(rec) else {}
|
|
437
|
+
ids = [q for q in side_ids(trial) if q not in have]
|
|
438
|
+
if not ids:
|
|
439
|
+
return rec
|
|
440
|
+
traj = trajectory.load(trial.trajectory_path)
|
|
441
|
+
if traj.error:
|
|
442
|
+
raise client.JudgeError(0, "the trajectory could not be read: " + traj.error)
|
|
443
|
+
return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _sides_todo(view) -> list:
|
|
447
|
+
"""(job, trial) for every side of an unjudged pair that lacks its score."""
|
|
448
|
+
out = []
|
|
449
|
+
for task, a, b in pairs_of(view):
|
|
450
|
+
for job, t in ((view.jobs[0], a), (view.jobs[1], b)):
|
|
451
|
+
if side_of(job, t) is None:
|
|
452
|
+
out.append((task, job, t))
|
|
453
|
+
return out
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
@app.route("/judge/compare", methods=["POST"])
|
|
457
|
+
def judge_compare_route():
|
|
458
|
+
"""Judge the unjudged pairs of this comparison - every side that lacks
|
|
459
|
+
its score - on the click of the button."""
|
|
460
|
+
from . import compare as compare_web
|
|
461
|
+
view = compare_web._comparison_requested()
|
|
462
|
+
if len(view.jobs) != 2:
|
|
463
|
+
abort(400, description="A judged comparison needs exactly two runs.")
|
|
464
|
+
sides = _sides_todo(view)
|
|
465
|
+
todo = list(dict.fromkeys(task for task, _, _ in sides))
|
|
466
|
+
_guard()
|
|
467
|
+
k = key()
|
|
468
|
+
done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})")
|
|
469
|
+
from flask import flash
|
|
470
|
+
flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
|
|
471
|
+
return redirect(url_for("compare", pick=view.references, cohort=view.cohort) + "#judgement")
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# --- feasibility triage, task-instruction quality
|
|
475
|
+
# Each over a small, purpose-built state (`state.control_state`,
|
|
476
|
+
# `task_state`); every count is code's; every answer is
|
|
477
|
+
# advisory beside the number, the verdict or the outcome it sits next to.
|
|
478
|
+
|
|
479
|
+
def _choice_row(q: str, a) -> dict:
|
|
480
|
+
"""A Choice's answer as a page shows it: the pick, its probability, the judge's confidence."""
|
|
481
|
+
row = row_of(q, a)
|
|
482
|
+
if a and a.get("choice"):
|
|
483
|
+
row["p"] = (a.get("probabilities") or {}).get(a["choice"])
|
|
484
|
+
return row
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
# -- feasibility triage: the control sides that did not behave -------------------------
|
|
488
|
+
|
|
489
|
+
def triage_sides(report) -> list:
|
|
490
|
+
"""(task_row, side) for every side that did not behave, with a trial to
|
|
491
|
+
read: nop accepted or breaking tests, oracle rejected, either failed
|
|
492
|
+
before the verifier. Missing sides have nothing to judge."""
|
|
493
|
+
out = []
|
|
494
|
+
for row in report.rows:
|
|
495
|
+
if row.verdict == feasibility.FEASIBLE:
|
|
496
|
+
continue
|
|
497
|
+
for s in (row.nop, row.oracle):
|
|
498
|
+
if not s.present:
|
|
499
|
+
continue
|
|
500
|
+
misbehaved = s.failed or s.broke_tests or (s.accepted if s.control == feasibility.NOP else s.rejected)
|
|
501
|
+
if misbehaved:
|
|
502
|
+
out.append((row, s))
|
|
503
|
+
return out
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _control_trial(side, jobs):
|
|
507
|
+
job = next((j for j in jobs if j.job_id == side.job_id), None)
|
|
508
|
+
trial = next((t for t in job.trials if t.trial_name == side.trial_name), None) if job else None
|
|
509
|
+
return job, trial
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def control_states(side, trial) -> dict:
|
|
513
|
+
traj = None
|
|
514
|
+
if trial is not None and trial.has_trajectory:
|
|
515
|
+
traj = trajectory.load(trial.trajectory_path)
|
|
516
|
+
if traj.error:
|
|
517
|
+
traj = None
|
|
518
|
+
build = lambda kind: judge_state.build(kind, traj, trial=trial, trial_dir=trial.dir if trial else None, control=side.control, outcome_label=side.label)
|
|
519
|
+
return batch.states_for(Q.group_ids("feasibility"), build)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def judge_control(side, jobs, api_key: str, post=None) -> dict | None:
|
|
523
|
+
"""Ask why one control side did not behave, over its own `control` state,
|
|
524
|
+
stored as that control trial's record. The verdict is not touched."""
|
|
525
|
+
job, trial = _control_trial(side, jobs)
|
|
526
|
+
if trial is None:
|
|
527
|
+
raise client.JudgeError(0, f"{side.control} trial {side.trial_name} is not in this workspace")
|
|
528
|
+
return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def triage_summary(report, jobs) -> dict:
|
|
532
|
+
"""For the feasibility page: per task and side the judged cause, the
|
|
533
|
+
counts, and what judging the rest would cost."""
|
|
534
|
+
d = ws_dir()
|
|
535
|
+
sides = triage_sides(report)
|
|
536
|
+
rows, judged, todo = {}, 0, []
|
|
537
|
+
for row, s in sides:
|
|
538
|
+
rec = store.load(d, s.job_id, s.trial_name) if d else None
|
|
539
|
+
a = ((rec.get("answers") or {}).get("control_failure_cause")) if store.is_current(rec) else None
|
|
540
|
+
if a:
|
|
541
|
+
rows.setdefault(row.task_base, {})[s.control] = _choice_row("control_failure_cause", a)
|
|
542
|
+
judged += 1
|
|
543
|
+
else:
|
|
544
|
+
todo.append((row, s))
|
|
545
|
+
return {"rows": rows, "sides": len(sides), "judged": judged, "unjudged": len(todo), "todo": todo, "keyed": bool(key()),
|
|
546
|
+
"question": Q.QUESTIONS["control_failure_cause"]["instructions"] if "control_failure_cause" in Q.QUESTIONS else ""}
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
# -- task-instruction quality: once per dataset and task ---------------------------------
|
|
550
|
+
|
|
551
|
+
def task_key(job) -> str:
|
|
552
|
+
"""The store's job slot for a dataset's task judgements: `task:<dataset>`,
|
|
553
|
+
so an instruction is judged once for every run of that dataset."""
|
|
554
|
+
return "task:" + (job.dataset or "unknown")
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def task_record(job, task_base: str) -> dict | None:
|
|
558
|
+
d = ws_dir()
|
|
559
|
+
rec = store.load(d, task_key(job), task_base) if d else None
|
|
560
|
+
return rec if store.is_current(rec) else None
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
@app.template_global()
|
|
564
|
+
def task_quality_of(job, trial) -> dict | None:
|
|
565
|
+
"""The instruction's score for a trial's task: the expectation, the level
|
|
566
|
+
it rounds to (display only), the judge's confidence; None until judged."""
|
|
567
|
+
rec = task_record(job, trial.task_base)
|
|
568
|
+
a = (rec.get("answers") or {}).get("instruction_quality") if rec else None
|
|
569
|
+
if not a or not isinstance(a.get("score"), (int, float)):
|
|
570
|
+
return None
|
|
571
|
+
legend = a.get("legend") or list(Q.QUESTIONS["instruction_quality"]["criteria"])
|
|
572
|
+
level = min(int(a["score"] + 0.5), len(legend) - 1)
|
|
573
|
+
return {"score": a["score"], "top": len(legend) - 1, "level": level, "label": legend[level], "legend": legend, "confidence": a.get("confidence"),
|
|
574
|
+
"asked_at": rec.get("asked_at"), "model": rec["judge"]["model"]}
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
@app.template_global()
|
|
578
|
+
def task_tab(job, trial, traj) -> dict:
|
|
579
|
+
"""The trial page's Task instruction tab: the score (None until judged),
|
|
580
|
+
the rubric it was scored on - the record's own legend once judged, the
|
|
581
|
+
current question's until then - and the instruction text as the agent
|
|
582
|
+
received it, which is what the judge read."""
|
|
583
|
+
quality = task_quality_of(job, trial)
|
|
584
|
+
rubric = quality["legend"] if quality else list(Q.QUESTIONS["instruction_quality"]["criteria"])
|
|
585
|
+
instruction = judge_state.instruction_of(traj) if traj is not None and not getattr(traj, "error", None) else ""
|
|
586
|
+
return {"quality": quality, "rubric": rubric, "instruction": instruction}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _task_instruction(trial) -> str:
|
|
590
|
+
"""The task's text as the agent received it, from the trial's trajectory."""
|
|
591
|
+
if not trial.has_trajectory:
|
|
592
|
+
return ""
|
|
593
|
+
traj = trajectory.load(trial.trajectory_path)
|
|
594
|
+
return "" if traj.error else judge_state.instruction_of(traj)
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def tasks_todo(job) -> list:
|
|
598
|
+
"""One trial per task of the job whose instruction is not yet judged and can be read."""
|
|
599
|
+
seen, out = set(), []
|
|
600
|
+
for t in job.trials:
|
|
601
|
+
if t.task_base in seen:
|
|
602
|
+
continue
|
|
603
|
+
seen.add(t.task_base)
|
|
604
|
+
if task_record(job, t.task_base) is None and t.has_trajectory:
|
|
605
|
+
out.append(t)
|
|
606
|
+
return out
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def task_summary(job) -> dict:
|
|
610
|
+
"""For the job page: how many of the job's tasks have their instruction judged."""
|
|
611
|
+
tasks = {t.task_base for t in job.trials}
|
|
612
|
+
judged = sum(1 for tb in tasks if task_record(job, tb) is not None)
|
|
613
|
+
return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())}
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def judge_task(job, trial, api_key: str, post=None) -> dict | None:
|
|
617
|
+
"""Score one task's instruction over the `task` state alone, stored under
|
|
618
|
+
the dataset's task slot - never under the trial."""
|
|
619
|
+
instruction = _task_instruction(trial)
|
|
620
|
+
if not instruction:
|
|
621
|
+
raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}")
|
|
622
|
+
states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction))
|
|
623
|
+
return batch.ask_and_store(ws_dir(), task_key(job), trial.task_base, states, api_key, post=post)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Task-instruction quality: the judge's score over a dataset's task text,
|
|
2
|
+
once per dataset and task, independent of any run. An ambiguous instruction
|
|
3
|
+
caps every agent on that task, which is a harness finding, not an agent one.
|
|
4
|
+
`/judge/tasks/<job_id>` (POST) asks the job's unjudged tasks on the user's
|
|
5
|
+
click; the score is shown beside the task on the job page and the trial
|
|
6
|
+
page, advisory, and changes no outcome."""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from flask import abort, flash, redirect, url_for
|
|
10
|
+
|
|
11
|
+
from .core import app
|
|
12
|
+
from . import shared
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@app.route("/judge/tasks/<job_id>", methods=["POST"])
|
|
16
|
+
def judge_tasks_route(job_id):
|
|
17
|
+
"""Ask the instruction score of every task of the job that has none, on the click of the button."""
|
|
18
|
+
from . import judge
|
|
19
|
+
job = shared._find_job(shared._jobs_for_request(), job_id)
|
|
20
|
+
if job is None:
|
|
21
|
+
abort(404)
|
|
22
|
+
todo = judge.tasks_todo(job)
|
|
23
|
+
judge._guard()
|
|
24
|
+
k = judge.key()
|
|
25
|
+
done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k), lambda t: t.task_base)
|
|
26
|
+
flash(f"Judged {done} of {len(todo)} task instruction(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
|
|
27
|
+
return redirect(url_for("job", job_id=job_id) + "#task-quality")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@app.template_global()
|
|
31
|
+
def task_quality_summary(job) -> dict:
|
|
32
|
+
from . import judge
|
|
33
|
+
return judge.task_summary(job)
|