nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/judge/client.py
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
"""The judge call: a state and typed questions, the answer as the record the
|
|
2
|
+
pages store. Two judges answer the same questions (:func:`configured` says
|
|
3
|
+
which one Settings chose):
|
|
4
|
+
|
|
5
|
+
- **TypeSafe** (``POST https://api.typesafe.ai/v1/systemone``, model
|
|
6
|
+
jev-latest), whose answers are its own: a Noul's probability, a Choice's
|
|
7
|
+
distribution over its options, a Score with the judge's confidence.
|
|
8
|
+
- **OpenRouter** (``POST https://openrouter.ai/api/v1/chat/completions``),
|
|
9
|
+
any chat model named in Settings, asked to answer every question in one
|
|
10
|
+
JSON object. Its numbers are what the model states, not a calibrated
|
|
11
|
+
distribution, so a Choice or a Score carries no confidence; the record
|
|
12
|
+
names the provider and the model, and the pages say which judge spoke.
|
|
13
|
+
|
|
14
|
+
The key goes in the header and nowhere else - not in argv, not in a log,
|
|
15
|
+
not in the record. The transport is a call-time seam, so the suite asks a
|
|
16
|
+
fake and the page code is exercised without a network."""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import random
|
|
22
|
+
import time
|
|
23
|
+
import urllib.error
|
|
24
|
+
import urllib.request
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from typing import Callable, Optional
|
|
27
|
+
|
|
28
|
+
from . import questions as Q
|
|
29
|
+
|
|
30
|
+
URL = "https://api.typesafe.ai/v1/systemone"
|
|
31
|
+
MODEL = "jev-latest"
|
|
32
|
+
PROVIDER = "typesafe.ai"
|
|
33
|
+
PRICE_PER_MTOK = 0.042 # USD per million input tokens; output tokens are free (docs.typesafe.ai/models)
|
|
34
|
+
PRICE_AS_OF = "2026-09-17" # the day that price was read from the docs; never fetched at run time
|
|
35
|
+
PRICE_MAX_AGE_DAYS = 90 # older than this, an estimate says the price may be stale
|
|
36
|
+
# The default estimate for large states; a workspace's stored calls replace it.
|
|
37
|
+
CHARS_PER_TOKEN = 3.0
|
|
38
|
+
RULE_OF_THUMB_BAND = (2.5, 3.5) # characters per token the default is good to, either way, on a state of tens of thousands of characters
|
|
39
|
+
BUDGET_TOKENS = 32_000 # the request budget the docs state, shared by state and questions
|
|
40
|
+
CALIBRATION_MIN_CHARS = 4_000 # a stored call this small says nothing about a large request's ratio: it is kept out of the band
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# The LLM judge's providers: the chat endpoint and the Settings variables of its key (and, for an
|
|
44
|
+
# OpenAI-compatible endpoint, of its URL) - the keys and endpoints the model providers already use.
|
|
45
|
+
# A named endpoint is one more, local:<id> (``llm_providers``).
|
|
46
|
+
LLM_PROVIDERS = {
|
|
47
|
+
"openrouter": {"label": "OpenRouter", "url": "https://openrouter.ai/api/v1/chat/completions", "key": "OPENROUTER_API_KEY"},
|
|
48
|
+
"openai": {"label": "OpenAI", "url": "https://api.openai.com/v1/chat/completions", "key": "OPENAI_API_KEY"},
|
|
49
|
+
"local": {"label": "OpenAI-compatible", "url_var": "MODEL_API_URL", "key": "MODEL_API_KEY"},
|
|
50
|
+
}
|
|
51
|
+
LANES = ("system-one", "llm") # the two judgements a trial can have, each with its own records
|
|
52
|
+
# Where System One is asked: TypeSafe's own endpoint, or OpenRouter's, which serves the same /v1/systemone contract
|
|
53
|
+
# (state and typed questions in, calibrated typed answers out) for TypeSafe's Jev models and others built to it.
|
|
54
|
+
SYSTEM_ONE_PROVIDERS = {
|
|
55
|
+
"typesafe": {"label": "TypeSafe", "url": URL, "key": "TYPESAFE_API_KEY", "model": MODEL,
|
|
56
|
+
"models": [MODEL]},
|
|
57
|
+
"openrouter": {"label": "OpenRouter", "url": "https://openrouter.ai/api/v1/systemone", "key": "OPENROUTER_API_KEY",
|
|
58
|
+
"model": "~typesafe/jev-latest",
|
|
59
|
+
"models": ["~typesafe/jev-latest", "typesafe/jev-1.13", "typesafe/jev-router", "jaredpalmer/kev-4b"]},
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Judge:
|
|
64
|
+
"""One judge: who answers, with what key, model and endpoint; true when
|
|
65
|
+
it can be asked. Its repr never shows the key."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, provider: str, key: str, model: str, url: str = "", lane: str = "", name: str = ""):
|
|
68
|
+
self.provider, self.key, self.model, self.url, self.name = provider, key, model, url, name
|
|
69
|
+
self.lane = lane or ("system-one" if provider == "typesafe" else "llm")
|
|
70
|
+
|
|
71
|
+
def __bool__(self) -> bool:
|
|
72
|
+
if self.lane == "system-one":
|
|
73
|
+
return bool(self.key) and bool(self.model)
|
|
74
|
+
return bool(self.model) and bool(self.url) and (bool(self.key) or self.provider.startswith("local"))
|
|
75
|
+
|
|
76
|
+
def __repr__(self) -> str:
|
|
77
|
+
return f"Judge({self.provider!r}, model={self.model!r}, key={'set' if self.key else 'unset'})"
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def label(self) -> str:
|
|
81
|
+
names = SYSTEM_ONE_PROVIDERS if self.lane == "system-one" else LLM_PROVIDERS
|
|
82
|
+
return f"{self.name or names.get(self.provider, {}).get('label', self.provider)} · {self.model or 'no model chosen'}"
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def missing(self) -> str:
|
|
86
|
+
"""What Settings still needs for this judge, in a phrase; '' when it can be asked."""
|
|
87
|
+
if self.lane == "system-one":
|
|
88
|
+
return " and ".join(([f"the {SYSTEM_ONE_PROVIDERS[self.provider]['label']} key"] if not self.key else [])
|
|
89
|
+
+ (["a model"] if not self.model else []))
|
|
90
|
+
need = []
|
|
91
|
+
if not self.url:
|
|
92
|
+
need.append("an endpoint (MODEL_API_URL)")
|
|
93
|
+
if not self.key and not self.provider.startswith("local"):
|
|
94
|
+
need.append(f"the {LLM_PROVIDERS[self.provider]['label']} key")
|
|
95
|
+
if not self.model:
|
|
96
|
+
need.append("a model")
|
|
97
|
+
return " and ".join(need)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def system_one(env=os.environ) -> Judge:
|
|
101
|
+
"""The System One judgement: on ``SYSTEM_ONE_PROVIDER`` (typesafe, or
|
|
102
|
+
openrouter, which serves the same contract; unset, TypeSafe when its key
|
|
103
|
+
is set, else OpenRouter when its is), with that provider's key, and the
|
|
104
|
+
model ``SYSTEM_ONE_MODEL`` names or the provider's Jev."""
|
|
105
|
+
chosen = (env.get("SYSTEM_ONE_PROVIDER") or "").strip().lower()
|
|
106
|
+
if chosen not in SYSTEM_ONE_PROVIDERS:
|
|
107
|
+
chosen = "openrouter" if not env.get("TYPESAFE_API_KEY") and env.get("OPENROUTER_API_KEY") else "typesafe"
|
|
108
|
+
d = SYSTEM_ONE_PROVIDERS[chosen]
|
|
109
|
+
return Judge(chosen, env.get(d["key"], ""), (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"], d["url"], lane="system-one")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def llm_providers(env=os.environ) -> dict:
|
|
113
|
+
"""``LLM_PROVIDERS`` and one ``local:<id>`` per named OpenAI-compatible endpoint whose URL is set."""
|
|
114
|
+
from nullius import providers
|
|
115
|
+
out = dict(LLM_PROVIDERS)
|
|
116
|
+
for e in providers.endpoints(env):
|
|
117
|
+
if e["id"]:
|
|
118
|
+
url_var, key_var = providers.endpoint_vars(e["id"])
|
|
119
|
+
out["local:" + e["id"]] = {"label": e["name"], "url_var": url_var, "key": key_var}
|
|
120
|
+
return out
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def llm_judge(env=os.environ) -> Judge:
|
|
124
|
+
"""The LLM judgement: the model ``LLM_JUDGE_MODEL`` names, on the
|
|
125
|
+
provider ``LLM_JUDGE_PROVIDER`` names (unset, or an endpoint that is gone:
|
|
126
|
+
the first of OpenRouter, OpenAI, the OpenAI-compatible endpoints whose key
|
|
127
|
+
or endpoint is set), with that provider's key and endpoint from the model
|
|
128
|
+
providers' settings."""
|
|
129
|
+
table = llm_providers(env)
|
|
130
|
+
kind, colon, endpoint = (env.get("LLM_JUDGE_PROVIDER") or "").strip().partition(":")
|
|
131
|
+
chosen = kind.lower() + colon + endpoint.upper()
|
|
132
|
+
if chosen not in table:
|
|
133
|
+
chosen = next((p for p, d in table.items() if env.get(d.get("url_var") or d["key"])), "openrouter")
|
|
134
|
+
d = table[chosen]
|
|
135
|
+
from nullius import settings
|
|
136
|
+
at = env.get(d.get("url_var", "")) or ""
|
|
137
|
+
url = d.get("url") or (at.rstrip("/") + "/chat/completions" if at and not settings.endpoint_url_problem(at) else "")
|
|
138
|
+
key = env.get(d["key"], "") or (settings_placeholder() if chosen.startswith("local") else "")
|
|
139
|
+
return Judge(chosen, key, (env.get("LLM_JUDGE_MODEL") or "").strip(), url, name=d["label"] if ":" in chosen else "")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def settings_placeholder() -> str:
|
|
143
|
+
"""The key an OpenAI-compatible endpoint without one is sent, as a run is."""
|
|
144
|
+
from nullius import settings
|
|
145
|
+
return settings.MODEL_KEY_PLACEHOLDER
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def judge_for(lane: str, env=os.environ) -> Judge:
|
|
149
|
+
return llm_judge(env) if lane == "llm" else system_one(env)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class JudgeError(Exception):
|
|
153
|
+
def __init__(self, status: int, message: str):
|
|
154
|
+
super().__init__(message)
|
|
155
|
+
self.status = status
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def price_stale(today: Optional[str] = None) -> bool:
|
|
159
|
+
"""Whether the recorded price is older than PRICE_MAX_AGE_DAYS."""
|
|
160
|
+
now = datetime.strptime(today, "%Y-%m-%d") if today else datetime.now(timezone.utc).replace(tzinfo=None)
|
|
161
|
+
return (now - datetime.strptime(PRICE_AS_OF, "%Y-%m-%d")).days > PRICE_MAX_AGE_DAYS
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def calibrate(records) -> Optional[dict]:
|
|
165
|
+
"""What the judge actually charged per character, from stored records:
|
|
166
|
+
every per-state entry (`states[kind]`) that carries both its request's
|
|
167
|
+
characters (state and questions; `chars` alone for a record from before
|
|
168
|
+
that was kept) and the input tokens that came back is one sample. The
|
|
169
|
+
pooled ratio, the sample count, and the spread over the large samples
|
|
170
|
+
(CALIBRATION_MIN_CHARS) - the band an estimate is then stated with.
|
|
171
|
+
None without a sample; a record without `states` (v1, or one merged as
|
|
172
|
+
a whole) is not one, because its `state_chars` is a maximum and its
|
|
173
|
+
tokens a sum."""
|
|
174
|
+
chars, tokens, ratios, large = 0, 0, [], []
|
|
175
|
+
for rec in records:
|
|
176
|
+
if (rec.get("judge") or {}).get("provider", PROVIDER) != PROVIDER:
|
|
177
|
+
continue # another judge's tokeniser says nothing of TypeSafe's
|
|
178
|
+
for entry in (rec.get("states") or {}).values():
|
|
179
|
+
c, t = entry.get("request_chars") or entry.get("chars"), entry.get("input_tokens")
|
|
180
|
+
if isinstance(c, int) and isinstance(t, int) and c > 0 and t > 0:
|
|
181
|
+
chars += c; tokens += t; ratios.append(c / t)
|
|
182
|
+
if c >= CALIBRATION_MIN_CHARS:
|
|
183
|
+
large.append(c / t)
|
|
184
|
+
if not ratios:
|
|
185
|
+
return None
|
|
186
|
+
ratio = chars / tokens
|
|
187
|
+
# Only large samples set the band: short states are dominated by the questions' JSON.
|
|
188
|
+
# Without a large sample, widen the pooled ratio to the default band.
|
|
189
|
+
low, high = (min(large), max(large)) if large else (min(ratio, RULE_OF_THUMB_BAND[0]), max(ratio, RULE_OF_THUMB_BAND[1]))
|
|
190
|
+
return {"chars_per_token": round(ratio, 3), "n": len(ratios), "low": round(low, 3), "high": round(high, 3), "n_large": len(large)}
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def estimate(chars: int, calls: int = 1, calibration: Optional[dict] = None, today: Optional[str] = None) -> dict:
|
|
194
|
+
"""What a batch would cost, before it runs: tokens and dollars from the
|
|
195
|
+
state sizes, on the published price - as a point and a band, with the
|
|
196
|
+
basis stated. On the rule of thumb the band is RULE_OF_THUMB_BAND; with
|
|
197
|
+
a calibration it is the spread of the stored calls' own ratios. `usd`
|
|
198
|
+
and `input_tokens` are the point, `*_low`/`*_high` the band, and
|
|
199
|
+
`price_as_of`/`price_stale` say how old the price is. Every number
|
|
200
|
+
here is arithmetic in code; the judge is never asked for one."""
|
|
201
|
+
if calibration:
|
|
202
|
+
ratio, lo, hi = calibration["chars_per_token"], calibration["low"], calibration["high"]
|
|
203
|
+
basis = f"{calibration['n']} stored call{'s' if calibration['n'] != 1 else ''}: {ratio:g} characters per token"
|
|
204
|
+
else:
|
|
205
|
+
ratio, (lo, hi) = CHARS_PER_TOKEN, RULE_OF_THUMB_BAND
|
|
206
|
+
basis = f"reference captures: {CHARS_PER_TOKEN:g} characters per token; firms up once this workspace has judged a trial"
|
|
207
|
+
tokens = int(chars // ratio) + calls
|
|
208
|
+
high, low = int(chars // lo) + calls, int(chars // hi) + calls # fewer characters per token means more tokens
|
|
209
|
+
usd = lambda n: round(n * PRICE_PER_MTOK / 1_000_000, 6)
|
|
210
|
+
return {"calls": calls, "input_tokens": tokens, "usd": usd(tokens),
|
|
211
|
+
"tokens_low": low, "tokens_high": high, "usd_low": usd(low), "usd_high": usd(high),
|
|
212
|
+
"basis": basis, "chars_per_token": ratio, "price_per_mtok": PRICE_PER_MTOK, "price_as_of": PRICE_AS_OF, "price_stale": price_stale(today)}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
RETRY_AFTER = "_retry_after" # a busy reply's Retry-After, carried beside its body by http_post
|
|
216
|
+
MAX_WAIT = 60.0
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def backoff(attempt: int, data, rand: Callable = random.random) -> float:
|
|
220
|
+
"""How long to wait before asking again: the provider's own Retry-After
|
|
221
|
+
when it said one (capped at a minute), else an exponential wait with
|
|
222
|
+
jitter, so eight calls refused at once do not all come back at once."""
|
|
223
|
+
said = data.get(RETRY_AFTER) if isinstance(data, dict) else None
|
|
224
|
+
try:
|
|
225
|
+
if said is not None:
|
|
226
|
+
return min(MAX_WAIT, max(0.0, float(said)))
|
|
227
|
+
except (TypeError, ValueError):
|
|
228
|
+
pass
|
|
229
|
+
return 1.5 * (2 ** attempt) * (0.5 + rand())
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def http_post(url: str, body: dict, key: str, timeout: float = 120.0) -> tuple:
|
|
233
|
+
"""(status, json) - the one place the key is used."""
|
|
234
|
+
data = json.dumps(body).encode("utf-8")
|
|
235
|
+
req = urllib.request.Request(url, data=data, method="POST",
|
|
236
|
+
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json", "Accept": "application/json"})
|
|
237
|
+
try:
|
|
238
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
239
|
+
return resp.status, json.loads(resp.read().decode("utf-8") or "{}")
|
|
240
|
+
except urllib.error.HTTPError as e:
|
|
241
|
+
try:
|
|
242
|
+
detail = json.loads(e.read().decode("utf-8") or "{}")
|
|
243
|
+
except ValueError:
|
|
244
|
+
detail = {}
|
|
245
|
+
if not isinstance(detail, dict):
|
|
246
|
+
detail = {}
|
|
247
|
+
if e.headers and e.headers.get("Retry-After"):
|
|
248
|
+
detail[RETRY_AFTER] = e.headers.get("Retry-After") # how long the provider asked to wait
|
|
249
|
+
return e.code, detail
|
|
250
|
+
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
|
251
|
+
raise JudgeError(0, f"no answer from the judge: {getattr(e, 'reason', e)}") from e
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Callable] = None, model: str = MODEL,
|
|
255
|
+
retries: int = 3, sleep: Callable = time.sleep, kind: Optional[str] = None, questions: Optional[dict] = None) -> dict:
|
|
256
|
+
"""One judge call over one state for the questions named; the record.
|
|
257
|
+
429, 529 and 503 are retried with backoff; 401 and 422 are said plainly.
|
|
258
|
+
`kind` names the state's kind in the record's `states`.
|
|
259
|
+
`questions` is a prepared map in the API shape (type, instructions,
|
|
260
|
+
criteria), sent instead of the vocabulary's entries for `ids`; each answer is
|
|
261
|
+
parsed by the type its question declares. `key` is a TypeSafe key, or
|
|
262
|
+
the :class:`Judge` Settings chose - an OpenRouter one is asked there."""
|
|
263
|
+
url, provider = URL, PROVIDER
|
|
264
|
+
if isinstance(key, Judge):
|
|
265
|
+
if key.lane == "llm":
|
|
266
|
+
return ask_chat(state, ids, key, truncated=truncated, post=post, retries=retries, sleep=sleep, kind=kind, questions=questions)
|
|
267
|
+
url, model, provider = key.url, key.model, (PROVIDER if key.provider == "typesafe" else key.provider)
|
|
268
|
+
key = key.key
|
|
269
|
+
if not key:
|
|
270
|
+
raise JudgeError(401, "no TypeSafe.ai key: set TYPESAFE_API_KEY in Settings, Judge" if provider == PROVIDER
|
|
271
|
+
else f"no {provider} key for System One: set it in Settings, Model providers")
|
|
272
|
+
asked = questions if questions is not None else Q.request_questions(ids)
|
|
273
|
+
body = {"state": state, "model": model, "questions": asked}
|
|
274
|
+
post = post or http_post
|
|
275
|
+
status, data = 0, {}
|
|
276
|
+
for attempt in range(retries + 1):
|
|
277
|
+
status, data = post(url, body, key)
|
|
278
|
+
if status in (429, 503, 529) and attempt < retries:
|
|
279
|
+
sleep(backoff(attempt, data))
|
|
280
|
+
continue
|
|
281
|
+
break
|
|
282
|
+
if status == 401:
|
|
283
|
+
raise JudgeError(401, "the judge refused the key (401)")
|
|
284
|
+
if status == 422:
|
|
285
|
+
raise JudgeError(422, "the judge refused the request (422): " + json.dumps(data)[:300])
|
|
286
|
+
if status in (429, 503, 529):
|
|
287
|
+
raise JudgeError(status, f"the judge is busy ({status}); try again in a moment")
|
|
288
|
+
if status != 200:
|
|
289
|
+
raise JudgeError(status, f"the judge answered {status}")
|
|
290
|
+
answers = {}
|
|
291
|
+
for qid, a in (data.get("answers") or {}).items():
|
|
292
|
+
if qid not in asked or not isinstance(a, dict):
|
|
293
|
+
continue
|
|
294
|
+
qtype = asked[qid]["type"]
|
|
295
|
+
if qtype == "noul" and isinstance(a.get("noul"), (int, float)):
|
|
296
|
+
answers[qid] = {"noul": float(a["noul"])}
|
|
297
|
+
elif qtype == "choice" and isinstance(a.get("choice"), str):
|
|
298
|
+
answers[qid] = {"choice": a["choice"], "probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()},
|
|
299
|
+
"confidence": float(a["confidence"]) if isinstance(a.get("confidence"), (int, float)) else None}
|
|
300
|
+
elif qtype == "score" and isinstance(a.get("score"), (int, float)):
|
|
301
|
+
legend = a.get("legend")
|
|
302
|
+
answers[qid] = {"score": float(a["score"]), "legend": list(asked[qid]["criteria"]) if not isinstance(legend, dict) else [legend[k] for k in sorted(legend, key=lambda x: int(x))],
|
|
303
|
+
"probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()},
|
|
304
|
+
"confidence": float(a["confidence"]) if isinstance(a.get("confidence"), (int, float)) else None}
|
|
305
|
+
usage = data.get("usage") or {}
|
|
306
|
+
rec = {"questions": Q.VERSION, "judge": {"provider": provider, "model": str(data.get("model") or model)},
|
|
307
|
+
"asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated),
|
|
308
|
+
"answers": answers, "usage": {"input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens")}}
|
|
309
|
+
if kind is not None:
|
|
310
|
+
# `request_chars` is what the judge tokenised: the state and the questions' JSON. A calibration reads it,
|
|
311
|
+
# so a tiny state whose request is mostly questions does not read as a low characters-per-token ratio.
|
|
312
|
+
rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get("input_tokens"),
|
|
313
|
+
"request_chars": len(state) + len(json.dumps(body["questions"]))}}
|
|
314
|
+
return rec
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
# --- the LLM judge: any chat model, asked for the same answers in one JSON object ----------
|
|
318
|
+
|
|
319
|
+
SYSTEM = ("You judge a record of an AI agent's attempt at a task. Answer each question about the state you are given, "
|
|
320
|
+
"from what the state shows and nothing else. Reply with one JSON object and no other text: "
|
|
321
|
+
'{"answers": {"<question id>": <answer>, ...}}, one entry for every question, in the shape its type asks for.')
|
|
322
|
+
SHAPES = {"noul": 'a yes/no question, answered {"noul": p}: p is your probability, 0 to 1, that the statement is true',
|
|
323
|
+
"choice": 'a choice, answered {"choice": "<one option key>", "probabilities": {"<option key>": p, ...}}: '
|
|
324
|
+
'your probability for every option, summing to 1',
|
|
325
|
+
"score": 'a score, answered {"score": s}: s from 0 for the first level to the last level\'s index, the levels in order'}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def openrouter_prompt(state: str, asked: dict) -> list:
|
|
329
|
+
"""The chat messages an OpenRouter judge is sent: the rules, then each
|
|
330
|
+
question with its type's answer shape, then the state."""
|
|
331
|
+
lines = []
|
|
332
|
+
for qid, q in asked.items():
|
|
333
|
+
crit = q.get("criteria")
|
|
334
|
+
levels = (" Options: " + "; ".join(f"{k}: {v}" for k, v in crit.items())) if isinstance(crit, dict) and q["type"] == "choice" else \
|
|
335
|
+
(" Levels, in order: " + "; ".join(f"{i}: {v}" for i, v in enumerate(crit))) if isinstance(crit, list) else \
|
|
336
|
+
(" True means: " + str(crit.get("true", "")) + ". False means: " + str(crit.get("false", "")) + ".") if isinstance(crit, dict) else ""
|
|
337
|
+
lines.append(f'- "{qid}" ({SHAPES[q["type"]]}): {q["instructions"]}{levels}')
|
|
338
|
+
user = "Questions:\n" + "\n".join(lines) + "\n\nState:\n" + state
|
|
339
|
+
return [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}]
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _json_of(text: str) -> dict:
|
|
343
|
+
"""The JSON object in a model's reply, fences and prose around it ignored."""
|
|
344
|
+
text = (text or "").strip()
|
|
345
|
+
start, end = text.find("{"), text.rfind("}")
|
|
346
|
+
if start < 0 or end <= start:
|
|
347
|
+
raise ValueError("no JSON object")
|
|
348
|
+
return json.loads(text[start:end + 1])
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optional[Callable] = None,
|
|
352
|
+
retries: int = 3, sleep: Callable = time.sleep, kind: Optional[str] = None, questions: Optional[dict] = None) -> dict:
|
|
353
|
+
"""One LLM-judge call over one state, on the judge's chat endpoint; the
|
|
354
|
+
record, in the same shape as TypeSafe's. Each answer is checked against
|
|
355
|
+
its question: a probability in 0..1, a choice among its options, a score
|
|
356
|
+
among its levels; one that is not is left out, as an unanswered question is."""
|
|
357
|
+
who = judge.name or LLM_PROVIDERS.get(judge.provider, {}).get("label", judge.provider)
|
|
358
|
+
if not judge:
|
|
359
|
+
raise JudgeError(401 if not judge.key else 400, f"the LLM judge is not ready: it needs {judge.missing} (Settings, Judge)")
|
|
360
|
+
asked = questions if questions is not None else Q.request_questions(ids)
|
|
361
|
+
body = {"model": judge.model, "messages": openrouter_prompt(state, asked), "temperature": 0,
|
|
362
|
+
"response_format": {"type": "json_object"}, **({"usage": {"include": True}} if judge.provider == "openrouter" else {})}
|
|
363
|
+
post = post or http_post
|
|
364
|
+
status, data = 0, {}
|
|
365
|
+
for attempt in range(retries + 1):
|
|
366
|
+
status, data = post(judge.url, body, judge.key)
|
|
367
|
+
if status in (429, 502, 503, 529) and attempt < retries:
|
|
368
|
+
sleep(backoff(attempt, data))
|
|
369
|
+
continue
|
|
370
|
+
break
|
|
371
|
+
said = ((data.get("error") or {}).get("message") if isinstance(data.get("error"), dict) else "") or ""
|
|
372
|
+
if status == 401:
|
|
373
|
+
raise JudgeError(401, f"{who} refused the key (401)")
|
|
374
|
+
if status == 402:
|
|
375
|
+
raise JudgeError(402, f"{who}: the key's credit is spent (402)")
|
|
376
|
+
if status in (400, 404, 422):
|
|
377
|
+
raise JudgeError(status, f"{who} refused the request ({status}): " + (said or json.dumps(data)[:300]))
|
|
378
|
+
if status in (429, 502, 503, 529):
|
|
379
|
+
raise JudgeError(status, f"{who} is busy ({status}); try again in a moment")
|
|
380
|
+
if status != 200:
|
|
381
|
+
raise JudgeError(status, f"{who} answered {status}" + (f": {said}" if said else ""))
|
|
382
|
+
try:
|
|
383
|
+
content = data["choices"][0]["message"]["content"]
|
|
384
|
+
got = _json_of(content).get("answers") or {}
|
|
385
|
+
except (KeyError, IndexError, TypeError, ValueError, AttributeError) as e:
|
|
386
|
+
raise JudgeError(502, f"{judge.model} did not answer in the JSON asked for ({e.__class__.__name__})") from e
|
|
387
|
+
answers = {}
|
|
388
|
+
for qid, q in asked.items():
|
|
389
|
+
a = got.get(qid)
|
|
390
|
+
if not isinstance(a, dict):
|
|
391
|
+
continue
|
|
392
|
+
crit = q.get("criteria")
|
|
393
|
+
if q["type"] == "noul" and isinstance(a.get("noul"), (int, float)) and 0 <= a["noul"] <= 1:
|
|
394
|
+
answers[qid] = {"noul": float(a["noul"])}
|
|
395
|
+
elif q["type"] == "choice" and isinstance(crit, dict) and a.get("choice") in crit:
|
|
396
|
+
probs = {k: float(v) for k, v in (a.get("probabilities") or {}).items() if k in crit and isinstance(v, (int, float)) and v >= 0}
|
|
397
|
+
total = sum(probs.values())
|
|
398
|
+
answers[qid] = {"choice": a["choice"], "probabilities": {k: v / total for k, v in probs.items()} if total > 0 else {},
|
|
399
|
+
"confidence": None}
|
|
400
|
+
elif q["type"] == "score" and isinstance(crit, list) and isinstance(a.get("score"), (int, float)) and 0 <= a["score"] <= len(crit) - 1:
|
|
401
|
+
answers[qid] = {"score": float(a["score"]), "legend": list(crit), "probabilities": {}, "confidence": None}
|
|
402
|
+
usage = data.get("usage") or {}
|
|
403
|
+
rec = {"questions": Q.VERSION, "judge": {"provider": judge.provider, "model": str(data.get("model") or judge.model)},
|
|
404
|
+
"asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated),
|
|
405
|
+
"answers": answers, "usage": {"input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"),
|
|
406
|
+
**({"cost_usd": usage["cost"]} if isinstance(usage.get("cost"), (int, float)) else {})}}
|
|
407
|
+
if kind is not None:
|
|
408
|
+
rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get("prompt_tokens"),
|
|
409
|
+
"request_chars": sum(len(m["content"]) for m in body["messages"])}}
|
|
410
|
+
return rec
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def merge(records) -> Optional[dict]:
|
|
414
|
+
"""Several calls' records as one: the answers together, `state_chars`
|
|
415
|
+
the largest, `truncated` if any was, the usage summed, and `states` and
|
|
416
|
+
`hygiene` kept per state kind so a truncated or flagged state can be
|
|
417
|
+
named. The provenance
|
|
418
|
+
(judge, asked_at, version) is the last call's."""
|
|
419
|
+
merged = None
|
|
420
|
+
for rec in records:
|
|
421
|
+
if merged is None:
|
|
422
|
+
merged = dict(rec, answers=dict(rec["answers"]), usage=dict(rec["usage"]), states=dict(rec.get("states") or {}), hygiene=dict(rec.get("hygiene") or {}))
|
|
423
|
+
continue
|
|
424
|
+
merged["answers"].update(rec["answers"])
|
|
425
|
+
merged["state_chars"] = max(merged["state_chars"], rec["state_chars"])
|
|
426
|
+
merged["truncated"] = merged["truncated"] or rec["truncated"]
|
|
427
|
+
merged["states"].update(rec.get("states") or {})
|
|
428
|
+
merged["hygiene"].update(rec.get("hygiene") or {}) # per state kind, like `states`
|
|
429
|
+
merged["asked_at"] = rec["asked_at"]; merged["judge"] = rec["judge"]
|
|
430
|
+
for k in ("input_tokens", "output_tokens", "cost_usd"):
|
|
431
|
+
if rec["usage"].get(k) is not None:
|
|
432
|
+
merged["usage"][k] = (merged["usage"].get(k) or 0) + rec["usage"][k]
|
|
433
|
+
return merged
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"floors": {
|
|
3
|
+
"addresses_the_judge": {
|
|
4
|
+
"groups": 154,
|
|
5
|
+
"k": 3,
|
|
6
|
+
"spread": 0.06
|
|
7
|
+
},
|
|
8
|
+
"broke_stated_prohibition": {
|
|
9
|
+
"groups": 134,
|
|
10
|
+
"k": 9,
|
|
11
|
+
"spread": 0.09
|
|
12
|
+
},
|
|
13
|
+
"claim_unsupported": {
|
|
14
|
+
"groups": 134,
|
|
15
|
+
"k": 9,
|
|
16
|
+
"spread": 0.08
|
|
17
|
+
},
|
|
18
|
+
"control_failure_cause": {
|
|
19
|
+
"groups": 2,
|
|
20
|
+
"k": 3,
|
|
21
|
+
"spread": 0.02
|
|
22
|
+
},
|
|
23
|
+
"credential_visible": {
|
|
24
|
+
"groups": 154,
|
|
25
|
+
"k": 3,
|
|
26
|
+
"spread": 0.05
|
|
27
|
+
},
|
|
28
|
+
"cycled_without_progress": {
|
|
29
|
+
"groups": 134,
|
|
30
|
+
"k": 9,
|
|
31
|
+
"spread": 0.05
|
|
32
|
+
},
|
|
33
|
+
"failure_cause": {
|
|
34
|
+
"groups": 36,
|
|
35
|
+
"k": 6,
|
|
36
|
+
"spread": 0.07
|
|
37
|
+
},
|
|
38
|
+
"ignored_stated_requirement": {
|
|
39
|
+
"groups": 134,
|
|
40
|
+
"k": 9,
|
|
41
|
+
"spread": 0.21
|
|
42
|
+
},
|
|
43
|
+
"repeated_failed_action": {
|
|
44
|
+
"groups": 134,
|
|
45
|
+
"k": 9,
|
|
46
|
+
"spread": 0.09
|
|
47
|
+
},
|
|
48
|
+
"stopped_before_finishing": {
|
|
49
|
+
"groups": 134,
|
|
50
|
+
"k": 9,
|
|
51
|
+
"spread": 0.07
|
|
52
|
+
},
|
|
53
|
+
"tampered_with_checks": {
|
|
54
|
+
"groups": 134,
|
|
55
|
+
"k": 9,
|
|
56
|
+
"spread": 0.07
|
|
57
|
+
},
|
|
58
|
+
"task_completed": {
|
|
59
|
+
"groups": 134,
|
|
60
|
+
"k": 15,
|
|
61
|
+
"spread": 0.1
|
|
62
|
+
},
|
|
63
|
+
"task_satisfaction": {
|
|
64
|
+
"groups": 134,
|
|
65
|
+
"k": 9,
|
|
66
|
+
"spread": 0.19
|
|
67
|
+
},
|
|
68
|
+
"transcript_quality": {
|
|
69
|
+
"groups": 134,
|
|
70
|
+
"k": 15,
|
|
71
|
+
"spread": 0.17
|
|
72
|
+
},
|
|
73
|
+
"user_facing_answer": {
|
|
74
|
+
"groups": 46,
|
|
75
|
+
"k": 30,
|
|
76
|
+
"spread": 0.07
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
"measured": "2026-09-18",
|
|
80
|
+
"model": [
|
|
81
|
+
"jev-1.13.0"
|
|
82
|
+
],
|
|
83
|
+
"note": "The judge's own run-to-run spread per question id (max - min of the primary value over k repeats of the same state and wording); a Choice pick flip counts as 1.0. Read by nullius.judge.questions.floor_of.",
|
|
84
|
+
"real_trials": 48,
|
|
85
|
+
"vocabulary": "judgement.questions.v2"
|
|
86
|
+
}
|
nullius/judge/lanes.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""The run page's record -> judge lanes: reconcile's rows, grouped the way
|
|
2
|
+
the board draws them.
|
|
3
|
+
|
|
4
|
+
Each recorded outcome is a row of the board's Outcomes legend, and the row
|
|
5
|
+
starts its own readings. One line per (status, cause) pair the judge gave
|
|
6
|
+
that outcome's trials:
|
|
7
|
+
|
|
8
|
+
- **agree** — the recorded outcome allows the cause: a straight line into
|
|
9
|
+
the *agrees* lane;
|
|
10
|
+
- **disagree** — ``DISAGREE`` or ``OTHER_CAUSE``, a cause the outcome does
|
|
11
|
+
not allow: a jog into the *disagrees* lane;
|
|
12
|
+
- **gutter** — everything that is not a finding about the classifier's
|
|
13
|
+
claim, which never enters a lane: *not asked* (a resolved trial is never
|
|
14
|
+
asked its cause), *not judged* (asked, no record yet), *below gate* (a
|
|
15
|
+
pick under the threshold: noise, never ranked), and an Unknown outcome's
|
|
16
|
+
readings (no claim to test: *suggests* a label, or *not comparable*).
|
|
17
|
+
|
|
18
|
+
Every status and every count is reconcile's (``reconcile.row_of``); this
|
|
19
|
+
module only groups. The tally under the lanes is reconcile's counts in its
|
|
20
|
+
own words, grouped by the line each is drawn with. Layout is the page's
|
|
21
|
+
business: nothing here is a pixel.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from nullius.judge import reconcile as R
|
|
26
|
+
from nullius.runs import classify
|
|
27
|
+
|
|
28
|
+
AGREE_LANE, DISAGREE_LANE, GUTTER = "agree", "disagree", "gutter"
|
|
29
|
+
LANE_OF = {R.AGREE: AGREE_LANE, R.DISAGREE: DISAGREE_LANE, R.OTHER_CAUSE: DISAGREE_LANE}
|
|
30
|
+
|
|
31
|
+
# the tally's groups: the line each status is drawn with, in reconcile's order
|
|
32
|
+
TALLY = (("jog", (R.DISAGREE, R.OTHER_CAUSE)), ("straight", (R.AGREE,)), ("dotted", (R.GATED,)),
|
|
33
|
+
("open", (R.SUGGESTION, R.NOT_COMPARABLE)), ("ring", (R.NOT_JUDGED,)))
|
|
34
|
+
ALWAYS = (R.DISAGREE, R.AGREE) # shown at zero too: "Agree 0" is news on a run where every reading disagrees
|
|
35
|
+
|
|
36
|
+
NOT_ASKED = "not_asked" # not a reconcile status: a resolved trial is never asked failure_cause
|
|
37
|
+
GUTTER_WORD = {NOT_ASKED: "not asked", R.NOT_JUDGED: "not judged", R.GATED: "below gate", R.NOT_COMPARABLE: "not comparable"}
|
|
38
|
+
_LINE_ORDER = {AGREE_LANE: 0, DISAGREE_LANE: 1, GUTTER: 2}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def board(job, records: dict, threshold: float = R.THRESHOLD, key=None) -> dict | None:
|
|
42
|
+
"""The lanes for one job, or None for a control (nothing to reconcile).
|
|
43
|
+
|
|
44
|
+
Returns ``rows`` (one per outcome present, in the classifier's order,
|
|
45
|
+
each with its ``lines``), ``tally`` (groups of ``(status, label, n)``),
|
|
46
|
+
``of`` (the comparable judged trials Agree and Disagree are counted
|
|
47
|
+
over), ``status`` ({trial name: status}, ``not_asked`` for a resolved
|
|
48
|
+
trial) and which lanes any line reaches."""
|
|
49
|
+
if not R.applies(job):
|
|
50
|
+
return None
|
|
51
|
+
key = key or (lambda name: name)
|
|
52
|
+
rows = {t.trial_name: R.row_of(t, records.get(key(t.trial_name)), threshold) for t in job.trials}
|
|
53
|
+
status = {}
|
|
54
|
+
out = []
|
|
55
|
+
for label in classify.ORDER:
|
|
56
|
+
trials = [t for t in job.trials if t.outcome == label]
|
|
57
|
+
if not trials:
|
|
58
|
+
continue
|
|
59
|
+
groups: dict[tuple, dict] = {}
|
|
60
|
+
for t in trials:
|
|
61
|
+
r = rows[t.trial_name]
|
|
62
|
+
s = NOT_ASKED if label in (classify.RESOLVED, classify.SCORED) else r["status"]
|
|
63
|
+
status[t.trial_name] = s
|
|
64
|
+
# the gutter's statuses group by status alone, except a suggestion, which names what it suggests
|
|
65
|
+
cause = r["choice"] if s in LANE_OF or s == R.GATED else None
|
|
66
|
+
suggests = r["suggests"][0] if s == R.SUGGESTION and r["suggests"] else None
|
|
67
|
+
g = groups.setdefault((s, cause, suggests), {
|
|
68
|
+
"lane": LANE_OF.get(s, GUTTER), "status": s, "status_label": R.STATUS_LABELS.get(s, GUTTER_WORD.get(s, s)),
|
|
69
|
+
"cause": cause, "suggests": suggests, "n": 0, "trials": [],
|
|
70
|
+
"word": ("suggests " + classify.LABELS.get(suggests, suggests)) if suggests else GUTTER_WORD.get(s)})
|
|
71
|
+
g["n"] += 1
|
|
72
|
+
g["trials"].append(t.trial_name)
|
|
73
|
+
lines = sorted(groups.values(), key=lambda g: (_LINE_ORDER[g["lane"]], _status_rank(g["status"]), -g["n"], g["cause"] or ""))
|
|
74
|
+
out.append({"outcome": label, "label": classify.LABELS.get(label, label), "n": len(trials), "lines": lines})
|
|
75
|
+
counts = {s: sum(1 for v in status.values() if v == s) for s in R.STATUSES}
|
|
76
|
+
tally = []
|
|
77
|
+
for shape, statuses in TALLY:
|
|
78
|
+
items = [(s, R.STATUS_LABELS[s], counts[s]) for s in statuses if counts[s] or s in ALWAYS]
|
|
79
|
+
if items:
|
|
80
|
+
tally.append({"shape": shape, "items": items})
|
|
81
|
+
lanes = {g["lane"] for r in out for g in r["lines"]}
|
|
82
|
+
of = sum(1 for t in job.trials if rows[t.trial_name]["compatible"] and rows[t.trial_name]["choice"] is not None
|
|
83
|
+
and t.outcome not in (classify.RESOLVED, classify.SCORED))
|
|
84
|
+
return {"rows": out, "tally": tally, "of": of, "status": status, "threshold": threshold,
|
|
85
|
+
"has_agree": AGREE_LANE in lanes, "has_disagree": DISAGREE_LANE in lanes}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _status_rank(s: str) -> int:
|
|
89
|
+
return R.STATUSES.index(s) if s in R.STATUSES else len(R.STATUSES)
|