nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""A trial: its trace, trajectory, conversation, artifacts and saved views."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
7
|
+
from collections import OrderedDict
|
|
8
|
+
from flask import abort, flash, jsonify, redirect, render_template, request, url_for
|
|
9
|
+
from markupsafe import Markup, escape
|
|
10
|
+
from nullius.traces import behavior, charts, conversation, trace, trajectory
|
|
11
|
+
from nullius.runs import artifacts, parse
|
|
12
|
+
|
|
13
|
+
from .core import app
|
|
14
|
+
from . import shared
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# What a saved trace view may be called. Defined in the data layer, because two
|
|
18
|
+
# things depend on agreeing about it: this route writes one, and ?trace= reads one.
|
|
19
|
+
SAVED_TRACE_RE = parse.TRACE_NAME_RE
|
|
20
|
+
SAVED_TRACE_DEFAULT = "trace-filtered.json"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def trace_file_path(trial, name: str) -> str | None:
|
|
24
|
+
"""Absolute path to a trace file inside *trial*'s ``agent/`` directory.
|
|
25
|
+
|
|
26
|
+
*name* arrives from a query string or a form field, so it is validated as a
|
|
27
|
+
filename rather than trusted as a path: the regex admits no separator and
|
|
28
|
+
`shared._safe_dest` refuses anything that still manages to leave the directory.
|
|
29
|
+
Returns None when the name is not one this app will read or write.
|
|
30
|
+
"""
|
|
31
|
+
if not SAVED_TRACE_RE.match((name or "").strip()):
|
|
32
|
+
return None
|
|
33
|
+
return shared._safe_dest(os.path.join(trial.dir, "agent"), name.strip())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _dataset_dirs(job):
|
|
37
|
+
"""Candidate dataset checkouts for a job's tasks, best lead first.
|
|
38
|
+
|
|
39
|
+
The job config's dataset path (absolute, or relative to wherever harbor
|
|
40
|
+
ran), then DATASET_ROOTS. Registry-fetched runs carry no path, so
|
|
41
|
+
the env var is the only way to point those at a local checkout.
|
|
42
|
+
"""
|
|
43
|
+
candidates = [job.dataset_path] + os.environ.get(
|
|
44
|
+
"DATASET_ROOTS", ""
|
|
45
|
+
).split(os.pathsep)
|
|
46
|
+
dirs = []
|
|
47
|
+
for c in candidates:
|
|
48
|
+
if not c:
|
|
49
|
+
continue
|
|
50
|
+
p = os.path.abspath(os.path.expanduser(c))
|
|
51
|
+
if os.path.isdir(p) and p not in dirs:
|
|
52
|
+
dirs.append(p)
|
|
53
|
+
return dirs
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _gold_localization(job, trial, traj):
|
|
57
|
+
"""Agent-edited files vs the task's gold patch, or None when no local
|
|
58
|
+
dataset checkout (or no parsable gold diff) can be found."""
|
|
59
|
+
if traj is None:
|
|
60
|
+
return None
|
|
61
|
+
for root in _dataset_dirs(job):
|
|
62
|
+
solve = os.path.join(root, trial.task_base, "solution", "solve.sh")
|
|
63
|
+
if not os.path.isfile(solve):
|
|
64
|
+
continue
|
|
65
|
+
try:
|
|
66
|
+
with open(solve, encoding="utf-8") as fh:
|
|
67
|
+
gold = behavior.gold_files(fh.read())
|
|
68
|
+
except OSError:
|
|
69
|
+
return None
|
|
70
|
+
if gold:
|
|
71
|
+
return behavior.localization(traj.files_touched, gold)
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _find_trial(job, trial_name):
|
|
76
|
+
return next((x for x in job.trials if x.trial_name == trial_name), None)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _trial_evidence(job_id, trial_name):
|
|
80
|
+
"""Resolve URL identifiers through parsed runs, never as filesystem paths."""
|
|
81
|
+
job = shared._find_job(shared._jobs_for_request(), job_id)
|
|
82
|
+
if job is None:
|
|
83
|
+
abort(404)
|
|
84
|
+
trial = _find_trial(job, trial_name)
|
|
85
|
+
if trial is None:
|
|
86
|
+
abort(404)
|
|
87
|
+
return job, trial
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _requested_trace(trial, *, notify=False):
|
|
91
|
+
"""One trace/filter resolution for the page and its read-only detail API.
|
|
92
|
+
|
|
93
|
+
`zoom` is parsed beside `hide`/`hide_span` because it is part of the same
|
|
94
|
+
URL-defined view — but it re-bases the waterfall's axis on one subtree and
|
|
95
|
+
hides nothing (`trace.Filter`).
|
|
96
|
+
"""
|
|
97
|
+
filt = trace.Filter.of(request.args.getlist("hide"), request.args.getlist("hide_span"),
|
|
98
|
+
request.args.get("zoom"))
|
|
99
|
+
wanted = (request.args.get("trace") or "").strip()
|
|
100
|
+
path, viewing = (trial.trace_path if trial.has_trace else None), ""
|
|
101
|
+
if wanted:
|
|
102
|
+
alt = trace_file_path(trial, wanted)
|
|
103
|
+
if alt and os.path.isfile(alt):
|
|
104
|
+
path, viewing = alt, os.path.basename(alt)
|
|
105
|
+
elif notify:
|
|
106
|
+
flash(Markup(f"No saved trace called <span class=\"mono\">{escape(wanted)}</span> "
|
|
107
|
+
"in this trial — showing the producer's own."), "error")
|
|
108
|
+
else:
|
|
109
|
+
abort(404)
|
|
110
|
+
return (_parsed_trace(path, filt) if path else None), viewing
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# Traces parsed for a view: every span-detail click, filter and zoom link asked
|
|
114
|
+
# for the whole fold, prune, redaction and breakdowns again (about a second at
|
|
115
|
+
# 6k spans). Kept by file (path, size, mtime) and view, a few at a time; a Trace
|
|
116
|
+
# is not changed once parsed.
|
|
117
|
+
_PARSED: "OrderedDict" = OrderedDict()
|
|
118
|
+
_PARSED_MAX = 8
|
|
119
|
+
_PARSED_LOCK = threading.Lock()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _parsed_trace(path, filt):
|
|
123
|
+
try:
|
|
124
|
+
st = os.stat(path)
|
|
125
|
+
except OSError:
|
|
126
|
+
return trace.load(path, filt)
|
|
127
|
+
key = (os.path.realpath(path), st.st_size, st.st_mtime_ns, filt)
|
|
128
|
+
with _PARSED_LOCK:
|
|
129
|
+
if key in _PARSED:
|
|
130
|
+
_PARSED.move_to_end(key)
|
|
131
|
+
return _PARSED[key]
|
|
132
|
+
tr = trace.load(path, filt)
|
|
133
|
+
with _PARSED_LOCK:
|
|
134
|
+
_PARSED[key] = tr
|
|
135
|
+
while len(_PARSED) > _PARSED_MAX:
|
|
136
|
+
_PARSED.popitem(last=False)
|
|
137
|
+
return tr
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@app.route("/trial/<job_id>/<trial_name>/artifact/<key>")
|
|
141
|
+
def artifact_page(job_id, trial_name, key):
|
|
142
|
+
job, trial = _trial_evidence(job_id, trial_name)
|
|
143
|
+
try:
|
|
144
|
+
record = artifacts.read(trial.dir, key, request.args.get("page", 1))
|
|
145
|
+
except (LookupError, OSError):
|
|
146
|
+
abort(404)
|
|
147
|
+
response = app.make_response(render_template("traces/artifact.html", job=job, trial=trial,
|
|
148
|
+
artifact=record))
|
|
149
|
+
response.headers["Cache-Control"] = "no-store"
|
|
150
|
+
return response
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@app.route("/trial/<job_id>/<trial_name>/conversation")
|
|
154
|
+
def conversation_data(job_id, trial_name):
|
|
155
|
+
_, t = _trial_evidence(job_id, trial_name)
|
|
156
|
+
traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None
|
|
157
|
+
payload = conversation.page(
|
|
158
|
+
traj, query=request.args.get("q", ""), kind=request.args.get("kind", "all"),
|
|
159
|
+
page=request.args.get("page", 1), position=request.args.get("position"),
|
|
160
|
+
step=request.args.get("step"), section=request.args.get("section"),
|
|
161
|
+
chunk=request.args.get("chunk", 0),
|
|
162
|
+
)
|
|
163
|
+
response = jsonify(payload)
|
|
164
|
+
response.headers["Cache-Control"] = "no-store"
|
|
165
|
+
return response
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@app.route("/trial/<job_id>/<trial_name>/trace-span")
|
|
169
|
+
def trace_span_data(job_id, trial_name):
|
|
170
|
+
_, t = _trial_evidence(job_id, trial_name)
|
|
171
|
+
tr, _ = _requested_trace(t)
|
|
172
|
+
try:
|
|
173
|
+
payload = conversation.span_detail(
|
|
174
|
+
tr, request.args.get("span", ""), section=request.args.get("section"),
|
|
175
|
+
chunk=request.args.get("chunk", 0),
|
|
176
|
+
)
|
|
177
|
+
except LookupError:
|
|
178
|
+
abort(404)
|
|
179
|
+
response = jsonify(payload)
|
|
180
|
+
response.headers["Cache-Control"] = "no-store"
|
|
181
|
+
return response
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@app.route("/trial/<job_id>/<trial_name>")
|
|
185
|
+
def trial(job_id, trial_name):
|
|
186
|
+
j, t = _trial_evidence(job_id, trial_name)
|
|
187
|
+
traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None
|
|
188
|
+
ok = traj is not None and not traj.error
|
|
189
|
+
bench_judge = bench_parts = None
|
|
190
|
+
if j.runner: # the benchmark's own judge: what it was asked, what it said
|
|
191
|
+
try:
|
|
192
|
+
with open(os.path.join(t.dir, "verifier", "judge.json"), encoding="utf-8") as fh:
|
|
193
|
+
bench_judge = json.load(fh)
|
|
194
|
+
except (OSError, ValueError):
|
|
195
|
+
pass
|
|
196
|
+
try: # an item scored part by part: each part's text, answer and score
|
|
197
|
+
with open(os.path.join(t.dir, "verifier", "parts.json"), encoding="utf-8") as fh:
|
|
198
|
+
got = json.load(fh)
|
|
199
|
+
if isinstance(got, dict) and isinstance(got.get("parts"), list):
|
|
200
|
+
bench_parts = {"says": str(got.get("says") or ""), "nouns": str(got.get("nouns") or "parts"),
|
|
201
|
+
"parts": [x for x in got["parts"] if isinstance(x, dict)]}
|
|
202
|
+
# what the definition chose to show of a part, under its own words, in its order
|
|
203
|
+
bench_parts["shows"] = list(dict.fromkeys(k for x in bench_parts["parts"] if isinstance(x.get("shows"), dict) for k in x["shows"]))
|
|
204
|
+
except (OSError, ValueError):
|
|
205
|
+
pass
|
|
206
|
+
loc = _gold_localization(j, t, traj) if ok else None
|
|
207
|
+
|
|
208
|
+
# `hide`/`hide_span` make a filtered view a plain link — shareable,
|
|
209
|
+
# bookmarkable, and the same thing `save_trace` writes out. `trace` opens a
|
|
210
|
+
# view saved earlier instead of the producer's own file.
|
|
211
|
+
# A second, narrower query key: which kinds each chart is drawing. Separate
|
|
212
|
+
# from `hide` because it changes one chart and not the trace — the waterfall,
|
|
213
|
+
# the breakdown and what `save_trace` writes are all blind to it.
|
|
214
|
+
cfilt = charts.ChartFilter.of(request.args.getlist("chart_off"))
|
|
215
|
+
tr, viewing = _requested_trace(t, notify=True)
|
|
216
|
+
# Over `tr.roots`, which `parse` has already pruned — the charts describe the
|
|
217
|
+
# filtered view by construction, so there is one definition of "which spans
|
|
218
|
+
# are in this view" and it is still the query string.
|
|
219
|
+
ch = charts.charts_for(tr.roots, cfilt) if tr and tr.visible else None
|
|
220
|
+
records = artifacts.listing(t.dir)
|
|
221
|
+
for record in records:
|
|
222
|
+
record["url"] = url_for("artifact_page", job_id=job_id, trial_name=trial_name,
|
|
223
|
+
key=record["key"], workspace=request.args.get("workspace"))
|
|
224
|
+
return render_template(
|
|
225
|
+
"traces/trial.html", job=j, trial=t, traj=traj, loc=loc, bench_judge=bench_judge, bench_parts=bench_parts,
|
|
226
|
+
trace=tr, charts=ch, viewing=viewing,
|
|
227
|
+
chart_filter=cfilt, default_save_name=SAVED_TRACE_DEFAULT,
|
|
228
|
+
artifacts=records,
|
|
229
|
+
judgement=_judgement_panel(j, t), llm_judgement=_judgement_panel(j, t, "llm"), csrf=shared._csrf_token(),
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _judgement_panel(job, trial, lane="system-one"):
|
|
234
|
+
from . import judge
|
|
235
|
+
return judge.panel(job, trial, lane)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@app.route("/trial/<job_id>/<trial_name>/save-trace", methods=["POST"])
|
|
239
|
+
def save_trace(job_id, trial_name):
|
|
240
|
+
"""Write the filtered view to a new trace file beside the trial's own.
|
|
241
|
+
|
|
242
|
+
The only route in this app that writes into a job directory. Three refusals
|
|
243
|
+
keep that safe: the name must be a plain `*.json` filename, the destination
|
|
244
|
+
must not exist, and it must not be the file being read — a producer's capture
|
|
245
|
+
is not reproducible, so nothing here may overwrite one.
|
|
246
|
+
"""
|
|
247
|
+
shared._require_csrf()
|
|
248
|
+
_, t = _trial_evidence(job_id, trial_name)
|
|
249
|
+
|
|
250
|
+
# The zoom rides along only so the page comes back as it was; the file written
|
|
251
|
+
# is the filtered tree whole — a zoom is a way of looking, not a change.
|
|
252
|
+
filt = trace.Filter.of(request.form.getlist("hide"), request.form.getlist("hide_span"),
|
|
253
|
+
request.form.get("zoom"))
|
|
254
|
+
# Filtering a saved view again is allowed, so the source is whichever file the
|
|
255
|
+
# page was rendering — falling back to the producer's if that one has gone.
|
|
256
|
+
source_name = (request.form.get("trace") or "").strip()
|
|
257
|
+
source = trace_file_path(t, source_name) if source_name else t.trace_path
|
|
258
|
+
if not source or not os.path.isfile(source):
|
|
259
|
+
source, source_name = t.trace_path, ""
|
|
260
|
+
back = url_for(
|
|
261
|
+
"trial", job_id=job_id, trial_name=trial_name,
|
|
262
|
+
trace=source_name or None, **filt.query,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
if not os.path.isfile(source):
|
|
266
|
+
flash("The trace this view was filtered from is no longer there.", "error")
|
|
267
|
+
return redirect(back)
|
|
268
|
+
if not filt.active:
|
|
269
|
+
flash("Nothing is filtered out — a saved view would just copy the original.", "error")
|
|
270
|
+
return redirect(back)
|
|
271
|
+
|
|
272
|
+
name = (request.form.get("filename") or "").strip() or SAVED_TRACE_DEFAULT
|
|
273
|
+
if not name.lower().endswith(".json"):
|
|
274
|
+
name += ".json"
|
|
275
|
+
dest = trace_file_path(t, name)
|
|
276
|
+
if not dest:
|
|
277
|
+
flash("Name it as a plain file ending in .json — letters, digits, dot, dash "
|
|
278
|
+
"and underscore, no directories.", "error")
|
|
279
|
+
return redirect(back)
|
|
280
|
+
shown = Markup(f'<span class="mono">{escape(os.path.basename(dest))}</span>')
|
|
281
|
+
if os.path.realpath(dest) == os.path.realpath(source) or os.path.basename(dest) == "trace.json":
|
|
282
|
+
flash(Markup(f"{shown} is the trace being read — save the view under another name."), "error")
|
|
283
|
+
return redirect(back)
|
|
284
|
+
if os.path.exists(dest):
|
|
285
|
+
flash(Markup(f"{shown} already exists; nothing was overwritten."), "error")
|
|
286
|
+
return redirect(back)
|
|
287
|
+
|
|
288
|
+
try:
|
|
289
|
+
with open(source, encoding="utf-8") as fh:
|
|
290
|
+
data = json.load(fh)
|
|
291
|
+
# Written from the raw document, so the saved file keeps every field the
|
|
292
|
+
# producer wrote — including the content this app redacts at render. It is
|
|
293
|
+
# exactly as sensitive as its source.
|
|
294
|
+
out = trace.filtered_document(data, filt)
|
|
295
|
+
with open(dest, "w", encoding="utf-8") as fh:
|
|
296
|
+
json.dump(out, fh, ensure_ascii=False, indent=1)
|
|
297
|
+
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
|
298
|
+
flash(Markup(f"Could not write {shown}: {escape(str(exc))}"), "error")
|
|
299
|
+
return redirect(back)
|
|
300
|
+
|
|
301
|
+
view = url_for("trial", job_id=job_id, trial_name=trial_name, trace=os.path.basename(dest))
|
|
302
|
+
flash(Markup(
|
|
303
|
+
f"Saved {shown} beside the trial's own trace — {out['filtered']['spans_removed']} "
|
|
304
|
+
f'span(s) removed, totals recomputed. <a href="{view}">Open it</a>.'
|
|
305
|
+
), "")
|
|
306
|
+
return redirect(back)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Workspaces: list, open, refresh, rename, delete."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from flask import flash, redirect, render_template, request, url_for
|
|
5
|
+
from markupsafe import Markup, escape
|
|
6
|
+
from nullius.runs import parse
|
|
7
|
+
from nullius import workspace
|
|
8
|
+
|
|
9
|
+
from .core import app
|
|
10
|
+
from . import shared
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@app.route("/workspaces")
|
|
15
|
+
def workspaces_page():
|
|
16
|
+
return render_template("workspaces/workspaces.html", workspaces=workspace.list_all(shared.WS_HOME),
|
|
17
|
+
home=shared.WS_HOME, csrf=shared._csrf_token())
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@app.route("/workspaces/open/<name>", methods=["POST"])
|
|
21
|
+
def workspace_open(name):
|
|
22
|
+
shared._require_csrf()
|
|
23
|
+
ws = workspace.load(shared.WS_HOME, name)
|
|
24
|
+
if ws is None:
|
|
25
|
+
flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
|
|
26
|
+
return redirect(url_for("workspaces_page"))
|
|
27
|
+
shared.open_workspace(ws)
|
|
28
|
+
return redirect(url_for("index"))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@app.route("/workspaces/refresh", methods=["POST"])
|
|
32
|
+
def workspace_refresh():
|
|
33
|
+
"""Throw the parse cache away and re-convert imported trees. The only way
|
|
34
|
+
a changed *foreign* run tree is noticed; a changed Harbor job is noticed
|
|
35
|
+
by its signature without this."""
|
|
36
|
+
shared._require_csrf()
|
|
37
|
+
ws = shared.STATE["workspace"]
|
|
38
|
+
if ws is None:
|
|
39
|
+
return redirect(url_for("index"))
|
|
40
|
+
shared.open_workspace(ws, refresh=True)
|
|
41
|
+
n = len(parse.find_job_dirs(shared.STATE["job_roots"]))
|
|
42
|
+
flash(Markup(f"Refreshed <span class=\"mono\">{escape(ws.label)}</span> — {n} run(s) "
|
|
43
|
+
"re-read; imported trees converted again."), "")
|
|
44
|
+
return redirect(request.referrer or url_for("index"))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.route("/workspaces/rename", methods=["POST"])
|
|
48
|
+
def workspace_rename():
|
|
49
|
+
"""Rename the workspace a row names, open or not: its label, and its
|
|
50
|
+
directory moved to the label's slug (``workspace.rename``). The open one
|
|
51
|
+
is read from where it is now. Takes the form token: a rename moves a
|
|
52
|
+
directory."""
|
|
53
|
+
shared._require_csrf()
|
|
54
|
+
open_ws = shared.STATE["workspace"]
|
|
55
|
+
name = request.form.get("name") or (open_ws.name if open_ws is not None else "")
|
|
56
|
+
label = (request.form.get("label") or "").strip()
|
|
57
|
+
is_open = open_ws is not None and open_ws.name == name
|
|
58
|
+
ws = open_ws if is_open else workspace.load(shared.WS_HOME, name)
|
|
59
|
+
if ws is None:
|
|
60
|
+
flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
|
|
61
|
+
elif label:
|
|
62
|
+
try:
|
|
63
|
+
workspace.rename(shared.WS_HOME, ws, label)
|
|
64
|
+
except OSError as exc:
|
|
65
|
+
flash(Markup(f'<span class="mono">{escape(ws.label)}</span> could not be renamed: {escape(exc.strerror or str(exc))}.'), "error")
|
|
66
|
+
return redirect(url_for("workspaces_page"))
|
|
67
|
+
if is_open:
|
|
68
|
+
shared.open_workspace(ws) # its imports moved with it: read from where it is now
|
|
69
|
+
return redirect(url_for("workspaces_page"))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@app.route("/workspaces/delete/<name>", methods=["POST"])
|
|
73
|
+
def workspace_delete(name):
|
|
74
|
+
"""Remove a workspace - its cache and imports, never the job roots it
|
|
75
|
+
read. The open one closes first. Takes the form token."""
|
|
76
|
+
shared._require_csrf()
|
|
77
|
+
ws = shared.STATE["workspace"]
|
|
78
|
+
if ws is not None and ws.name == name:
|
|
79
|
+
shared.close_workspace()
|
|
80
|
+
if workspace.delete(shared.WS_HOME, name):
|
|
81
|
+
flash(Markup(f'Removed workspace <span class="mono">{escape(name)}</span>; its job roots are untouched.'), "")
|
|
82
|
+
else:
|
|
83
|
+
flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
|
|
84
|
+
return redirect(url_for("workspaces_page"))
|
nullius/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Nullius's own backend: everything the application does with a run.
|
|
2
|
+
|
|
3
|
+
This package owns the whole path from a directory of benchmark output to a
|
|
4
|
+
rendered page, one package per purpose:
|
|
5
|
+
|
|
6
|
+
- `runs/` — reading one Harbor job tree: `parse`, `classify`, `live` (the
|
|
7
|
+
feed a running job appends), `artifacts`, `feasibility`, `launches`,
|
|
8
|
+
`verifier` (a verifier's report, read as its catalogue entry describes it),
|
|
9
|
+
and the `cli`.
|
|
10
|
+
- `traces/` — what a trial recorded: `trace`, `trajectory`,
|
|
11
|
+
`conversation`, `charts`, `behavior`, `otel`.
|
|
12
|
+
- `compare/` — runs against runs.
|
|
13
|
+
- `catalogue/` — benchmarks and datasets: a folder per benchmark under
|
|
14
|
+
`benchmarks/` (`benchmark.json`, its datasets and report block nested in it,
|
|
15
|
+
and its `standard.json` when it runs on the Nullius standard).
|
|
16
|
+
- `recipes/` — `launch`, the vocabulary the run builder composes.
|
|
17
|
+
- `providers` — where a run's model calls go, and the builder's steps.
|
|
18
|
+
- `harbor/` — `source`: the Harbor a recipe pins, built here, kept, scanned.
|
|
19
|
+
- `importers/` — run trees other harnesses write: `atif`, `otel`, and the
|
|
20
|
+
`cli`.
|
|
21
|
+
- `judge/` — judgement: `questions`, `state`, `client`, `store`, `batch`,
|
|
22
|
+
`reconcile`, `lanes`.
|
|
23
|
+
- `execution`, `hosts`, `gitrefs` — running a recipe here or on a host, the
|
|
24
|
+
hosts and their keys, a repository's branches and tags.
|
|
25
|
+
- `workspace`, `settings`, `redact` — cross-cutting, at the top.
|
|
26
|
+
|
|
27
|
+
**These are Nullius's own implementations**, each with the tests that pin it;
|
|
28
|
+
nothing in this application installs another package to run.
|
|
29
|
+
|
|
30
|
+
One name, one implementation: `app/tests/test_dependency_surface.py` counts
|
|
31
|
+
providers so a local package cannot shadow an installed distribution.
|
|
32
|
+
"""
|
nullius/__main__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.agent/v1",
|
|
3
|
+
"key": "codex",
|
|
4
|
+
"label": "Codex",
|
|
5
|
+
"summary": "OpenAI's coding agent.",
|
|
6
|
+
"upstream": {
|
|
7
|
+
"license": "Apache-2.0"
|
|
8
|
+
},
|
|
9
|
+
"start": {
|
|
10
|
+
"acp": [
|
|
11
|
+
"{install}/node_modules/.bin/codex-acp"
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
"model": {
|
|
15
|
+
"apis": [
|
|
16
|
+
"responses"
|
|
17
|
+
],
|
|
18
|
+
"env": {
|
|
19
|
+
"CODEX_HOME": "{home}/.codex",
|
|
20
|
+
"NULLIUS_RELAY_KEY": "{key}"
|
|
21
|
+
},
|
|
22
|
+
"files": {
|
|
23
|
+
"{home}/.codex/config.toml": "config.toml"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"caches": [
|
|
27
|
+
"{home}/.codex/.tmp"
|
|
28
|
+
],
|
|
29
|
+
"options": {
|
|
30
|
+
"reasoning_effort": {
|
|
31
|
+
"summary": "How much reasoning the model uses before answering.",
|
|
32
|
+
"values": [
|
|
33
|
+
"minimal",
|
|
34
|
+
"low",
|
|
35
|
+
"medium",
|
|
36
|
+
"high"
|
|
37
|
+
],
|
|
38
|
+
"default": "medium"
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"default": "2.2.2",
|
|
42
|
+
"versions": {
|
|
43
|
+
"2.2.2": {
|
|
44
|
+
"upstream": {
|
|
45
|
+
"package": "@agentclientprotocol/codex-acp",
|
|
46
|
+
"core": "@openai/codex 0.160.1, a dependency of the adapter (^0.160.1), pinned in install"
|
|
47
|
+
},
|
|
48
|
+
"install": {
|
|
49
|
+
"npm": [
|
|
50
|
+
"@agentclientprotocol/codex-acp@2.2.2",
|
|
51
|
+
"@openai/codex@0.160.1"
|
|
52
|
+
]
|
|
53
|
+
},
|
|
54
|
+
"env": {
|
|
55
|
+
"INITIAL_AGENT_MODE": "agent-full-access"
|
|
56
|
+
},
|
|
57
|
+
"fidelity": {
|
|
58
|
+
"command": [
|
|
59
|
+
"{install}/node_modules/.bin/codex",
|
|
60
|
+
"exec",
|
|
61
|
+
"--skip-git-repo-check",
|
|
62
|
+
"--json",
|
|
63
|
+
"--",
|
|
64
|
+
"{prompt}"
|
|
65
|
+
]
|
|
66
|
+
},
|
|
67
|
+
"proven": {
|
|
68
|
+
"agent.task/v1": {
|
|
69
|
+
"passed": "3/3",
|
|
70
|
+
"on": "2026-11-02",
|
|
71
|
+
"model": "openai/gpt-5-mini",
|
|
72
|
+
"provider": "openrouter",
|
|
73
|
+
"version": "2.2.2",
|
|
74
|
+
"kit_version": 1,
|
|
75
|
+
"backend": "docker",
|
|
76
|
+
"job": "2026-11-02__13-16-33__kit__agent-task-v1__codex-2.2.2"
|
|
77
|
+
},
|
|
78
|
+
"agent.workspace/v1": {
|
|
79
|
+
"passed": "2/2",
|
|
80
|
+
"on": "2026-11-02",
|
|
81
|
+
"model": "openai/gpt-5-mini",
|
|
82
|
+
"provider": "openrouter",
|
|
83
|
+
"version": "2.2.2",
|
|
84
|
+
"kit_version": 1,
|
|
85
|
+
"backend": "docker",
|
|
86
|
+
"job": "2026-11-02__13-17-34__kit__agent-workspace-v1__codex-2.2.2"
|
|
87
|
+
},
|
|
88
|
+
"agent.conversation/v1": {
|
|
89
|
+
"passed": "2/2",
|
|
90
|
+
"on": "2026-11-02",
|
|
91
|
+
"model": "openai/gpt-5-mini",
|
|
92
|
+
"provider": "openrouter",
|
|
93
|
+
"version": "2.2.2",
|
|
94
|
+
"kit_version": 1,
|
|
95
|
+
"backend": "docker",
|
|
96
|
+
"job": "2026-11-02__13-20-14__kit__agent-conversation-v1__codex-2.2.2"
|
|
97
|
+
},
|
|
98
|
+
"fidelity": {
|
|
99
|
+
"verdict": "same",
|
|
100
|
+
"on": "2026-10-12",
|
|
101
|
+
"model": "gpt-5-mini",
|
|
102
|
+
"provider": "openai",
|
|
103
|
+
"version": "2.2.2",
|
|
104
|
+
"job": "2026-10-12__16-11-01__fidelity__codex-2.2.2",
|
|
105
|
+
"passes": 2,
|
|
106
|
+
"differs": []
|
|
107
|
+
},
|
|
108
|
+
"agent.tools/v1": {
|
|
109
|
+
"passed": "2/2",
|
|
110
|
+
"on": "2026-11-02",
|
|
111
|
+
"model": "openai/gpt-5-mini",
|
|
112
|
+
"provider": "openrouter",
|
|
113
|
+
"version": "2.2.2",
|
|
114
|
+
"kit_version": 2,
|
|
115
|
+
"backend": "docker",
|
|
116
|
+
"job": "2026-11-02__13-22-34__kit__agent-tools-v1__codex-2.2.2",
|
|
117
|
+
"skills": {
|
|
118
|
+
"decoys": {
|
|
119
|
+
"passed": "1/1",
|
|
120
|
+
"ok": true
|
|
121
|
+
},
|
|
122
|
+
"chain": {
|
|
123
|
+
"passed": "1/1",
|
|
124
|
+
"ok": true
|
|
125
|
+
},
|
|
126
|
+
"retry": {
|
|
127
|
+
"passed": "1/1",
|
|
128
|
+
"ok": true
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
"agent.search/v1": {
|
|
133
|
+
"passed": "3/3",
|
|
134
|
+
"on": "2026-11-12",
|
|
135
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
136
|
+
"provider": "openrouter",
|
|
137
|
+
"version": "2.2.2",
|
|
138
|
+
"kit_version": 1,
|
|
139
|
+
"backend": "docker",
|
|
140
|
+
"job": "2026-11-12__10-24-26__kit__agent-search-v1__codex-2.2.2"
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
"0.16.0": {
|
|
145
|
+
"install": {
|
|
146
|
+
"npm": [
|
|
147
|
+
"@zed-industries/codex-acp@0.16.0"
|
|
148
|
+
]
|
|
149
|
+
},
|
|
150
|
+
"proven": {
|
|
151
|
+
"agent.task/v1": {
|
|
152
|
+
"passed": "3/3",
|
|
153
|
+
"on": "2026-11-02",
|
|
154
|
+
"model": "openai/gpt-5-mini",
|
|
155
|
+
"provider": "openrouter",
|
|
156
|
+
"version": "0.16.0",
|
|
157
|
+
"kit_version": 1,
|
|
158
|
+
"backend": "docker",
|
|
159
|
+
"job": "2026-11-02__13-20-41__kit__agent-task-v1__codex-0.16.0"
|
|
160
|
+
},
|
|
161
|
+
"agent.workspace/v1": {
|
|
162
|
+
"passed": "2/2",
|
|
163
|
+
"on": "2026-11-02",
|
|
164
|
+
"model": "openai/gpt-5-mini",
|
|
165
|
+
"provider": "openrouter",
|
|
166
|
+
"version": "0.16.0",
|
|
167
|
+
"kit_version": 1,
|
|
168
|
+
"backend": "docker",
|
|
169
|
+
"job": "2026-11-02__13-21-58__kit__agent-workspace-v1__codex-0.16.0"
|
|
170
|
+
},
|
|
171
|
+
"agent.conversation/v1": {
|
|
172
|
+
"passed": "2/2",
|
|
173
|
+
"on": "2026-11-02",
|
|
174
|
+
"model": "openai/gpt-5-mini",
|
|
175
|
+
"provider": "openrouter",
|
|
176
|
+
"version": "0.16.0",
|
|
177
|
+
"kit_version": 1,
|
|
178
|
+
"backend": "docker",
|
|
179
|
+
"job": "2026-11-02__13-24-39__kit__agent-conversation-v1__codex-0.16.0"
|
|
180
|
+
},
|
|
181
|
+
"agent.tools/v1": {
|
|
182
|
+
"passed": "2/2",
|
|
183
|
+
"on": "2026-11-02",
|
|
184
|
+
"model": "openai/gpt-5-mini",
|
|
185
|
+
"provider": "openrouter",
|
|
186
|
+
"version": "0.16.0",
|
|
187
|
+
"kit_version": 2,
|
|
188
|
+
"backend": "docker",
|
|
189
|
+
"job": "2026-11-02__13-22-38__kit__agent-tools-v1__codex-0.16.0",
|
|
190
|
+
"skills": {
|
|
191
|
+
"decoys": {
|
|
192
|
+
"passed": "1/1",
|
|
193
|
+
"ok": true
|
|
194
|
+
},
|
|
195
|
+
"chain": {
|
|
196
|
+
"passed": "1/1",
|
|
197
|
+
"ok": true
|
|
198
|
+
},
|
|
199
|
+
"retry": {
|
|
200
|
+
"passed": "1/1",
|
|
201
|
+
"ok": true
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
},
|
|
205
|
+
"agent.search/v1": {
|
|
206
|
+
"passed": "3/3",
|
|
207
|
+
"on": "2026-11-12",
|
|
208
|
+
"model": "meta/muse-spark-1.3-contributor",
|
|
209
|
+
"provider": "openrouter",
|
|
210
|
+
"version": "0.16.0",
|
|
211
|
+
"kit_version": 1,
|
|
212
|
+
"backend": "docker",
|
|
213
|
+
"job": "2026-11-12__11-08-33__kit__agent-search-v1__codex-0.16.0"
|
|
214
|
+
}
|
|
215
|
+
},
|
|
216
|
+
"upstream": {
|
|
217
|
+
"package": "@zed-industries/codex-acp",
|
|
218
|
+
"core": "openai/codex rust-v0.137.0 (f221438b691b8f749d98f22077c93ebe01923fbe), compiled into the adapter's binary"
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Codex's configuration for a Nullius run, written into the home it is given.
|
|
2
|
+
# Its model calls go to the relay, a provider of its own: Codex's built-in
|
|
3
|
+
# OpenAI provider does not follow OPENAI_BASE_URL and would call OpenAI
|
|
4
|
+
# directly. The relay holds the key. Codex runs without its own sandbox
|
|
5
|
+
# (which needs bubblewrap) and without asking: the run is the isolation, and
|
|
6
|
+
# no one is there to answer. The reasoning effort is always written (the
|
|
7
|
+
# option reasoning_effort, its default Codex's own): left out, codex-acp and
|
|
8
|
+
# Codex's CLI would each fill it in their own way. Its plugins are off: with
|
|
9
|
+
# them on, Codex 0.160 fetches OpenAI's curated plugins repository (about
|
|
10
|
+
# 103 MB) into .codex/.tmp/plugins at every start - a download per unit, into
|
|
11
|
+
# every unit's home - and a run gives it no plugin to use.
|
|
12
|
+
model = "{model}"
|
|
13
|
+
model_reasoning_effort = "{opt_reasoning_effort}"
|
|
14
|
+
model_provider = "nullius"
|
|
15
|
+
approval_policy = "never"
|
|
16
|
+
sandbox_mode = "danger-full-access"
|
|
17
|
+
|
|
18
|
+
[features]
|
|
19
|
+
plugins = false
|
|
20
|
+
|
|
21
|
+
[model_providers.nullius]
|
|
22
|
+
name = "Nullius relay"
|
|
23
|
+
base_url = "{base_url}"
|
|
24
|
+
env_key = "NULLIUS_RELAY_KEY"
|
|
25
|
+
wire_api = "responses"
|