nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""OpenTelemetry span exports: a directory of OTLP/JSON files in, one job out.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.importers.cli traces-export/ -o jobs_imported
|
|
4
|
+
|
|
5
|
+
A production trace export is not a benchmark run — no task, no verifier, no
|
|
6
|
+
reward — but it is spans with clocks, which is what the trial page's waterfall
|
|
7
|
+
and ``/charts`` draw. So this lays the export out as the one shape the app
|
|
8
|
+
walks: a job directory holding **one trial per trace id**, each with the
|
|
9
|
+
``harbor-trace-v1`` document ``traces/otel.py`` builds under ``agent/``. The
|
|
10
|
+
format, the conventions read and the field mapping are that module's; this one
|
|
11
|
+
decides what a trial *is* and what its ``result.json`` may honestly say.
|
|
12
|
+
|
|
13
|
+
What it says is little, on purpose:
|
|
14
|
+
|
|
15
|
+
- **The outcome is unknown, and no reward exists.** A span in ERROR marks that
|
|
16
|
+
span in the trace; it is not an exception on the trial, because a failed
|
|
17
|
+
call is not a failed task and the app's outcome taxonomy is about tasks.
|
|
18
|
+
``exception_info`` and ``verifier_result`` are therefore empty, which
|
|
19
|
+
``classify`` reads as *unknown* — not resolved, not failed.
|
|
20
|
+
- **Tokens are the trace's own totals** — the sum over its LLM spans — written
|
|
21
|
+
only when at least one span reported usage; a trace with no usage attributes
|
|
22
|
+
records no ``agent_result`` at all rather than three zeros.
|
|
23
|
+
- **The model is the first LLM span's**, the agent the resource's
|
|
24
|
+
``service.name``. An export that names neither shows neither.
|
|
25
|
+
- **The clock is the trace's window**, first span start to last span end, as
|
|
26
|
+
both the trial's ``started_at``/``finished_at`` and its ``agent_execution``
|
|
27
|
+
phase — the only phase there is evidence for.
|
|
28
|
+
|
|
29
|
+
Recognition is by content: a ``.json`` file whose top level is an
|
|
30
|
+
``ExportTraceServiceRequest`` (``resourceSpans``), or a JSON-lines file of them
|
|
31
|
+
— under the directory at any depth. What one job is under a root the app opened is this module's answer
|
|
32
|
+
(``trees``): an export directory is one job, and a parent that holds no
|
|
33
|
+
export of its own but several export directories is one job per directory —
|
|
34
|
+
each is one producer's export (the trial's agent is its ``service.name``),
|
|
35
|
+
and pooling them would put several agents under one heading. The CLI
|
|
36
|
+
converts the directory it is given as one job; a parent given to it is one
|
|
37
|
+
pooled job, which is the one place the two still read a tree differently.
|
|
38
|
+
Nothing Harbor writes carries ``resourceSpans``, and an ATIF
|
|
39
|
+
trajectory carries ``schema_version`` instead, so nothing here claims their
|
|
40
|
+
directories.
|
|
41
|
+
|
|
42
|
+
A file the reader refuses — bad JSON, no ``resourceSpans``, an unsupported
|
|
43
|
+
schema URL — is reported with its redacted message and skipped; only when
|
|
44
|
+
*nothing* converts is that fatal. The source directory is never written to:
|
|
45
|
+
everything lands under the output root, and a re-convert rewrites the same
|
|
46
|
+
names in place (a trace the export no longer holds is removed), so refreshing
|
|
47
|
+
a workspace updates rather than duplicates.
|
|
48
|
+
"""
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import json
|
|
52
|
+
import os
|
|
53
|
+
import re
|
|
54
|
+
import shutil
|
|
55
|
+
from datetime import datetime, timezone
|
|
56
|
+
|
|
57
|
+
from nullius.redact import redact
|
|
58
|
+
from nullius.traces import otel as reader
|
|
59
|
+
from nullius.traces import trace
|
|
60
|
+
|
|
61
|
+
NAME = "otel"
|
|
62
|
+
LABEL = "OpenTelemetry"
|
|
63
|
+
|
|
64
|
+
# Directory-name characters for a trial: the root span's name can be anything
|
|
65
|
+
# a producer chose, and a slash in it would nest the trial.
|
|
66
|
+
_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def find_exports(root: str) -> list[str]:
|
|
71
|
+
"""Every OTLP/JSON file under *root*, sorted. A cheap head check first,
|
|
72
|
+
then the real read: the walk runs on every workspace open, over roots
|
|
73
|
+
that may hold thousands of Harbor files that are not this."""
|
|
74
|
+
out = []
|
|
75
|
+
for dirpath, _dirnames, filenames in os.walk(root):
|
|
76
|
+
for name in filenames:
|
|
77
|
+
path = os.path.join(dirpath, name)
|
|
78
|
+
if not (name.endswith(".json") or name.endswith(".jsonl")):
|
|
79
|
+
continue
|
|
80
|
+
if not reader.looks_like_export(path):
|
|
81
|
+
continue
|
|
82
|
+
try:
|
|
83
|
+
if reader.read_file(path):
|
|
84
|
+
out.append(path)
|
|
85
|
+
except reader.ReadError:
|
|
86
|
+
continue
|
|
87
|
+
return sorted(out)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def detect(path: str) -> bool:
|
|
91
|
+
"""Does *path* hold at least one readable OTLP/JSON trace export?"""
|
|
92
|
+
return os.path.isdir(path) and bool(find_exports(path))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def trees(root: str) -> list[str]:
|
|
96
|
+
"""The directories under *root* that are one job each.
|
|
97
|
+
|
|
98
|
+
*root* itself when an export file sits directly in it — an export
|
|
99
|
+
directory is one job however its files are arranged below. Otherwise
|
|
100
|
+
each child directory holding an export, one job per producer's export
|
|
101
|
+
rather than one pooled job of several agents. Nothing when there is no
|
|
102
|
+
export under *root* at all.
|
|
103
|
+
"""
|
|
104
|
+
exports = find_exports(root) if os.path.isdir(root) else []
|
|
105
|
+
if not exports:
|
|
106
|
+
return []
|
|
107
|
+
if any(os.path.dirname(path) == root for path in exports):
|
|
108
|
+
return [root]
|
|
109
|
+
return [os.path.join(root, name) for name in sorted(os.listdir(root))
|
|
110
|
+
if os.path.isdir(os.path.join(root, name)) and detect(os.path.join(root, name))]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def refused(root: str) -> list[str]:
|
|
114
|
+
"""(path, redacted reason) for every candidate file the reader refuses —
|
|
115
|
+
what the CLI reports beside the job it wrote."""
|
|
116
|
+
out = []
|
|
117
|
+
for dirpath, _dirnames, filenames in os.walk(root):
|
|
118
|
+
for name in sorted(filenames):
|
|
119
|
+
path = os.path.join(dirpath, name)
|
|
120
|
+
if not (name.endswith(".json") or name.endswith(".jsonl")):
|
|
121
|
+
continue
|
|
122
|
+
if not reader.looks_like_export(path):
|
|
123
|
+
continue
|
|
124
|
+
try:
|
|
125
|
+
reader.read_file(path)
|
|
126
|
+
except reader.ReadError as exc:
|
|
127
|
+
# The reader names the file by basename; this names it by its
|
|
128
|
+
# path under the root, once, in place of that.
|
|
129
|
+
reason = str(exc)
|
|
130
|
+
prefix = os.path.basename(path) + ": "
|
|
131
|
+
if reason.startswith(prefix):
|
|
132
|
+
reason = reason[len(prefix):]
|
|
133
|
+
out.append(f"{os.path.relpath(path, root)}: {reason}")
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _iso(ns: int) -> str:
|
|
139
|
+
return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _dump(path: str, data) -> None:
|
|
143
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
144
|
+
json.dump(data, fh, ensure_ascii=False, indent=1)
|
|
145
|
+
fh.write("\n")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _window(document: dict) -> tuple:
|
|
149
|
+
"""(start_ns, end_ns) over every span, end None when no span ended."""
|
|
150
|
+
starts, ends = [], []
|
|
151
|
+
for raw in trace.raw_spans_of(document.get("spans") or []):
|
|
152
|
+
starts.append(raw["start_ns"])
|
|
153
|
+
if raw.get("end_ns") is not None:
|
|
154
|
+
ends.append(raw["end_ns"])
|
|
155
|
+
return (min(starts) if starts else None, max(ends) if ends else None)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def trial_name_of(document: dict) -> str:
|
|
159
|
+
"""``<root span name>__<7 hex of the trace id>`` — readable, and unique
|
|
160
|
+
per trace the way a Harbor trial name is unique per attempt."""
|
|
161
|
+
tid = str(document.get("session_id") or "")
|
|
162
|
+
stem = _UNSAFE.sub("_", str(document.get("trial_name") or "")).strip("._-") or "trace"
|
|
163
|
+
return f"{stem[:80]}__{tid[:7] or 'unknown'}"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def write_trial(document: dict, job_dir: str) -> dict:
|
|
167
|
+
"""One trace document as a trial directory. Returns a summary."""
|
|
168
|
+
trial = trial_name_of(document)
|
|
169
|
+
trial_dir = os.path.join(job_dir, trial)
|
|
170
|
+
agent_dir = os.path.join(trial_dir, "agent")
|
|
171
|
+
os.makedirs(agent_dir, exist_ok=True)
|
|
172
|
+
|
|
173
|
+
start_ns, end_ns = _window(document)
|
|
174
|
+
started = _iso(start_ns) if start_ns else None
|
|
175
|
+
finished = _iso(end_ns) if end_ns else None
|
|
176
|
+
tokens = (document.get("totals") or {}).get("tokens") or {}
|
|
177
|
+
model = reader.model_of(document)
|
|
178
|
+
agent = document.get("agent") or {}
|
|
179
|
+
|
|
180
|
+
result = {
|
|
181
|
+
"trial_name": trial,
|
|
182
|
+
"task_name": f"{NAME}/{document.get('trial_name') or trial}",
|
|
183
|
+
"source": NAME,
|
|
184
|
+
"agent_info": {
|
|
185
|
+
"name": agent.get("name") or "",
|
|
186
|
+
"version": agent.get("version") or "",
|
|
187
|
+
"model_info": {"name": model},
|
|
188
|
+
},
|
|
189
|
+
# Deliberately empty: a span's status is not a task's outcome, and no
|
|
190
|
+
# verifier ran. The walker reads this pair as `unknown`.
|
|
191
|
+
"exception_info": {},
|
|
192
|
+
"verifier_result": {},
|
|
193
|
+
"started_at": started,
|
|
194
|
+
"finished_at": finished,
|
|
195
|
+
"environment_setup": None,
|
|
196
|
+
"agent_setup": None,
|
|
197
|
+
"agent_execution": (
|
|
198
|
+
{"started_at": started, "finished_at": finished} if started and finished else None),
|
|
199
|
+
"verifier": None,
|
|
200
|
+
"config": {"environment": {"type": ""}},
|
|
201
|
+
}
|
|
202
|
+
if tokens:
|
|
203
|
+
result["agent_result"] = {
|
|
204
|
+
"n_input_tokens": tokens.get("input_tokens", 0),
|
|
205
|
+
"n_output_tokens": tokens.get("output_tokens", 0),
|
|
206
|
+
"n_cache_tokens": tokens.get("cache_read_tokens", 0),
|
|
207
|
+
"cost_usd": None,
|
|
208
|
+
}
|
|
209
|
+
_dump(os.path.join(trial_dir, "result.json"), result)
|
|
210
|
+
_dump(os.path.join(agent_dir, "trace.json"), document)
|
|
211
|
+
return {"trial_dir": trial, "started_at": started, "finished_at": finished,
|
|
212
|
+
"model": model, "agent": agent.get("name") or ""}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _write_job(documents: list, out_root: str, job_name: str, command: str,
|
|
216
|
+
root: str, skipped: list) -> str:
|
|
217
|
+
job_dir = os.path.join(out_root, job_name)
|
|
218
|
+
os.makedirs(job_dir, exist_ok=True)
|
|
219
|
+
summaries = [write_trial(doc, job_dir) for doc in documents]
|
|
220
|
+
|
|
221
|
+
# Derived output this importer wholly owns: a trial the export no longer
|
|
222
|
+
# produces is stale and would keep being counted.
|
|
223
|
+
produced = {s["trial_dir"] for s in summaries}
|
|
224
|
+
for name in os.listdir(job_dir):
|
|
225
|
+
sub = os.path.join(job_dir, name)
|
|
226
|
+
if os.path.isdir(sub) and name not in produced:
|
|
227
|
+
shutil.rmtree(sub)
|
|
228
|
+
|
|
229
|
+
starts = sorted(s["started_at"] for s in summaries if s["started_at"])
|
|
230
|
+
ends = sorted(s["finished_at"] for s in summaries if s["finished_at"])
|
|
231
|
+
_dump(os.path.join(job_dir, "result.json"), {
|
|
232
|
+
"started_at": starts[0] if starts else None,
|
|
233
|
+
"finished_at": ends[-1] if ends else None,
|
|
234
|
+
"n_total_trials": len(summaries),
|
|
235
|
+
"stats": {"n_errored_trials": 0, "n_running_trials": 0,
|
|
236
|
+
"n_pending_trials": 0, "n_retries": 0},
|
|
237
|
+
})
|
|
238
|
+
agents = sorted({s["agent"] for s in summaries if s["agent"]}) or [""]
|
|
239
|
+
models = sorted({s["model"] for s in summaries if s["model"]})
|
|
240
|
+
_dump(os.path.join(job_dir, "config.json"), {
|
|
241
|
+
"job_name": job_name,
|
|
242
|
+
"harness": NAME,
|
|
243
|
+
"command": command,
|
|
244
|
+
"agents": [{"name": agents[0], "model_name": models[0] if len(models) == 1 else "",
|
|
245
|
+
"kwargs": {}}],
|
|
246
|
+
"datasets": [{"name": NAME, "path": root, "n_tasks": len(summaries)}],
|
|
247
|
+
"import": {"format": reader.FORMAT, "conventions": reader.CONVENTIONS,
|
|
248
|
+
"skipped": skipped},
|
|
249
|
+
})
|
|
250
|
+
return job_dir
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def convert(root: str, out_root: str, job_name: str = "",
|
|
254
|
+
command: str = "") -> list[str]:
|
|
255
|
+
"""Convert every OTLP/JSON export under *root* into one job directory.
|
|
256
|
+
|
|
257
|
+
Traces are grouped across files, one trial each. Files the reader refuses
|
|
258
|
+
are listed in the job's ``config.json`` under ``import.skipped`` with their
|
|
259
|
+
redacted reasons; when nothing reads, that list is the error.
|
|
260
|
+
"""
|
|
261
|
+
paths = find_exports(root)
|
|
262
|
+
skipped = refused(root)
|
|
263
|
+
if not paths:
|
|
264
|
+
raise SystemExit(redact(
|
|
265
|
+
f"no OTLP/JSON trace exports (resourceSpans) found in {root}"
|
|
266
|
+
+ ("; refused: " + "; ".join(skipped) if skipped else "")))
|
|
267
|
+
try:
|
|
268
|
+
documents = reader.read(paths)
|
|
269
|
+
except reader.ReadError as exc: # a file changed under us since find_exports
|
|
270
|
+
raise SystemExit(str(exc)) from None
|
|
271
|
+
if not documents:
|
|
272
|
+
raise SystemExit(redact(f"the OTLP/JSON exports in {root} carry no spans"))
|
|
273
|
+
name = job_name or f"{NAME}__{os.path.basename(os.path.abspath(root)) or 'export'}"
|
|
274
|
+
return [_write_job(documents, out_root, name, command, root, skipped)]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Judgement: structured, confidence-scored answers about a recorded trial
|
|
2
|
+
from a judge model (TypeSafe.ai System One), shown beside the deterministic
|
|
3
|
+
evidence and never in its place.
|
|
4
|
+
|
|
5
|
+
The contract: the question
|
|
6
|
+
vocabulary (``questions.py``, one id per question, versioned as a set), the
|
|
7
|
+
state each question reads (``state.py``, built from the redacted views only,
|
|
8
|
+
truncated head-and-tail with a marker), the record shape (``client.py``),
|
|
9
|
+
and where a judgement lives - the workspace store, keyed by trial and
|
|
10
|
+
question version (``store.py``), never under the job's own tree. A surface
|
|
11
|
+
asks through ``batch.py``: the questions grouped by state, one call per
|
|
12
|
+
state, the records merged and stored - so no page groups or merges on its
|
|
13
|
+
own.
|
|
14
|
+
"""
|
nullius/judge/batch.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Asking a set of questions over the states they read, and keeping the
|
|
2
|
+
answers: the seam every surface uses, so a page never groups questions by
|
|
3
|
+
state or merges records on its own.
|
|
4
|
+
|
|
5
|
+
``states_for(ids, build)`` groups the ids by the state kind each reads and
|
|
6
|
+
builds each state once; ``ask_states`` makes one call per state and merges
|
|
7
|
+
the records (``client.merge``); ``ask_and_store`` does that and saves the
|
|
8
|
+
result under the trial's key in the workspace store, merged over what was
|
|
9
|
+
there. A new surface registers its group (``questions.register``) and, if
|
|
10
|
+
its questions read a state of their own, that state's builder
|
|
11
|
+
(``state.register``), then calls these with its own key.
|
|
12
|
+
|
|
13
|
+
Every call also carries the ``hygiene`` group - does the state address the
|
|
14
|
+
judge, does it still show a credential - because state is untrusted text
|
|
15
|
+
from a job directory and output tokens are free. Their answers never land
|
|
16
|
+
in ``answers``: they go under ``hygiene[kind]`` in the record, one reading
|
|
17
|
+
per state kind, so a surface shows the flag beside every answer drawn from
|
|
18
|
+
that state and no answer changes because of it."""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
from typing import Callable, Optional
|
|
23
|
+
|
|
24
|
+
from . import client, questions as Q, store
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def with_hygiene(ids) -> list:
|
|
28
|
+
"""The ids a call sends: the surface's own, then the hygiene group."""
|
|
29
|
+
return list(ids) + [q for q in Q.hygiene_ids() if q not in ids]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def split_hygiene(rec: dict, kind: str) -> dict:
|
|
33
|
+
"""Move the hygiene answers out of `rec["answers"]` into
|
|
34
|
+
`rec["hygiene"][kind]` - `{id: probability}` - so the surface's answers
|
|
35
|
+
read as before and the reading is kept per state kind."""
|
|
36
|
+
readings = {}
|
|
37
|
+
for q in Q.hygiene_ids():
|
|
38
|
+
a = rec["answers"].pop(q, None)
|
|
39
|
+
if a and isinstance(a.get("noul"), (int, float)):
|
|
40
|
+
readings[q] = a["noul"]
|
|
41
|
+
if readings:
|
|
42
|
+
rec.setdefault("hygiene", {})[kind] = readings
|
|
43
|
+
return rec
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def states_for(ids, build: Callable) -> dict:
|
|
47
|
+
"""{kind: (ids, text, truncated)}: the questions grouped by the state
|
|
48
|
+
they read, each state built once by `build(kind) -> (text, truncated)`."""
|
|
49
|
+
groups: dict = {}
|
|
50
|
+
for q in ids:
|
|
51
|
+
groups.setdefault(Q.state_of(q), []).append(q)
|
|
52
|
+
out = {}
|
|
53
|
+
for kind, qs in groups.items():
|
|
54
|
+
text, truncated = build(kind)
|
|
55
|
+
out[kind] = (qs, text, truncated)
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def preflight(states: dict, calibration: Optional[dict] = None, budget: int = client.BUDGET_TOKENS) -> dict:
|
|
60
|
+
"""Refuse locally a call the judge would refuse: each state's text plus
|
|
61
|
+
its questions' JSON, estimated at the upper edge of the band, against
|
|
62
|
+
the request budget. Raises JudgeError(413) naming the state kind and
|
|
63
|
+
roughly how far over it is; returns {kind: tokens_high} otherwise.
|
|
64
|
+
|
|
65
|
+
state.LIMIT is 60,000 characters - about 20,000 tokens at the measured
|
|
66
|
+
3 per token, 24,000 at the band's edge of 2.5 - so a built-in state fits on the default
|
|
67
|
+
of thumb. The check matters when a calibration says the judge counts
|
|
68
|
+
fewer characters per token than that, and for a state kind a later
|
|
69
|
+
group registers with a limit of its own."""
|
|
70
|
+
out = {}
|
|
71
|
+
for kind, (qs, text, _) in states.items():
|
|
72
|
+
if not qs:
|
|
73
|
+
continue
|
|
74
|
+
chars = len(text) + len(json.dumps(Q.request_questions(with_hygiene(qs))))
|
|
75
|
+
est = client.estimate(chars, calls=1, calibration=calibration)
|
|
76
|
+
out[kind] = est["tokens_high"]
|
|
77
|
+
if est["tokens_high"] > budget:
|
|
78
|
+
over = est["tokens_high"] - budget
|
|
79
|
+
raise client.JudgeError(413, f"the `{kind}` state is too big for one judge call: up to {est['tokens_high']:,} tokens "
|
|
80
|
+
f"({chars:,} characters, {est['basis']}) against a budget of {budget:,}; "
|
|
81
|
+
f"about {over:,} tokens - roughly {int(over * est['chars_per_token']):,} characters - over. Nothing was sent.")
|
|
82
|
+
return out
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def ask_states(states: dict, key: str, post: Optional[Callable] = None, calibration: Optional[dict] = None,
|
|
86
|
+
budget: int = client.BUDGET_TOKENS) -> Optional[dict]:
|
|
87
|
+
"""One judge call per state, the records merged into one; None when
|
|
88
|
+
there was nothing to ask. `preflight` runs first, so a batch the judge
|
|
89
|
+
would refuse is refused here before any call - and before any charge.
|
|
90
|
+
Each call carries the hygiene group; its readings are kept per state
|
|
91
|
+
kind under `hygiene`, apart from `answers`."""
|
|
92
|
+
preflight(states, calibration=calibration, budget=budget)
|
|
93
|
+
records = []
|
|
94
|
+
for kind, (qs, text, truncated) in states.items():
|
|
95
|
+
if qs:
|
|
96
|
+
records.append(split_hygiene(client.ask(text, with_hygiene(qs), key, truncated=truncated, post=post, kind=kind), kind))
|
|
97
|
+
merged = client.merge(records)
|
|
98
|
+
if merged is not None:
|
|
99
|
+
hygiene = {}
|
|
100
|
+
for rec in records:
|
|
101
|
+
hygiene.update(rec.get("hygiene") or {})
|
|
102
|
+
merged["hygiene"] = hygiene
|
|
103
|
+
return merged
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, post: Optional[Callable] = None,
|
|
107
|
+
calibration: Optional[dict] = None) -> Optional[dict]:
|
|
108
|
+
"""`ask_states`, then the merged record saved under the trial's key -
|
|
109
|
+
its answers over the version's earlier ones. Returns the record as
|
|
110
|
+
stored. The pre-flight reads the workspace's own calibration unless one
|
|
111
|
+
is given, so a judge that counts more tokens per character than the rule
|
|
112
|
+
of thumb is refused on its own evidence."""
|
|
113
|
+
if calibration is None:
|
|
114
|
+
calibration = store.calibration(ws_dir)
|
|
115
|
+
rec = ask_states(states, key, post=post, calibration=calibration)
|
|
116
|
+
if rec is not None:
|
|
117
|
+
# a call over some of the kinds keeps the earlier readings for the others, as the answers are kept
|
|
118
|
+
lane = getattr(key, "lane", "system-one") # the LLM judge's records apart from System One's
|
|
119
|
+
earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane) or {}
|
|
120
|
+
rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})}
|
|
121
|
+
store.save(ws_dir, job_id, trial_name, rec, lane=lane)
|
|
122
|
+
return rec
|