nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""What the pages share: the open workspace, its jobs, the catalogue cache, path resolution, the query helpers, the form-token guard."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import glob
|
|
5
|
+
import hashlib
|
|
6
|
+
import ipaddress
|
|
7
|
+
from itertools import islice
|
|
8
|
+
import secrets
|
|
9
|
+
import os
|
|
10
|
+
from urllib.parse import urlsplit
|
|
11
|
+
from flask import abort, flash, g, has_request_context, request, session
|
|
12
|
+
from markupsafe import Markup, escape
|
|
13
|
+
from werkzeug.exceptions import Forbidden
|
|
14
|
+
from nullius import importers as harnesses
|
|
15
|
+
from nullius import execution
|
|
16
|
+
from nullius.catalogue import catalog
|
|
17
|
+
from nullius import workspace
|
|
18
|
+
from nullius import settings, paths
|
|
19
|
+
from nullius.redact import redact
|
|
20
|
+
|
|
21
|
+
from .core import app
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_APP_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # app/, one up from this package
|
|
25
|
+
# User settings, loaded at start and read again before any request once it has
|
|
26
|
+
# changed on disk (settings.sync), so a key written by hand while the app
|
|
27
|
+
# runs counts from the next request, without a restart; existing environment
|
|
28
|
+
# variables win. The Settings page edits the same file (nullius.settings
|
|
29
|
+
# knows every variable read here).
|
|
30
|
+
ENV_FILE = paths.settings_file()
|
|
31
|
+
settings.load(ENV_FILE)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _host_name(host: str) -> str:
|
|
35
|
+
"""A Host header's name without its port: "[::1]:5057" is "::1"."""
|
|
36
|
+
if host.startswith("["):
|
|
37
|
+
return host[1:host.find("]")] if "]" in host else host
|
|
38
|
+
return host.rsplit(":", 1)[0] if host.count(":") == 1 else host
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _served_name(name: str) -> bool:
|
|
42
|
+
"""A name this server answers to. A page on a name that resolves here
|
|
43
|
+
(DNS rebinding) would otherwise be same-origin with the app and pass the
|
|
44
|
+
loopback, Origin and form-token guards alike; an address literal or
|
|
45
|
+
localhost cannot be such a page's origin. The --host name it was started
|
|
46
|
+
with is accepted as well."""
|
|
47
|
+
name = name.lower().rstrip(".")
|
|
48
|
+
if name == "localhost" or name.endswith(".localhost") or name == str(app.config.get("SERVED_HOST", "")).lower():
|
|
49
|
+
return True
|
|
50
|
+
try:
|
|
51
|
+
ipaddress.ip_address(name)
|
|
52
|
+
return True
|
|
53
|
+
except ValueError:
|
|
54
|
+
return False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@app.before_request
|
|
58
|
+
def _refuse_foreign_host():
|
|
59
|
+
if not _served_name(_host_name(request.host)):
|
|
60
|
+
abort(421, description="This server answers on its own address only (localhost, an IP address or its --host name).")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@app.before_request
|
|
64
|
+
def _sync_env_file():
|
|
65
|
+
"""The file as it is now: a hand edit applies to this request and every
|
|
66
|
+
one after it. A file that cannot be read leaves the settings as loaded."""
|
|
67
|
+
try:
|
|
68
|
+
settings.sync(ENV_FILE)
|
|
69
|
+
except (OSError, ValueError, UnicodeDecodeError) as e:
|
|
70
|
+
app.logger.warning("%s not read: %s", os.path.basename(ENV_FILE), getattr(e, "strerror", None) or e)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# What each benchmark actually measures. Keyed by the dataset name Harbor
|
|
74
|
+
# records in the job config; unknown datasets fall back to a stub so a new
|
|
75
|
+
# benchmark shows up in the UI rather than vanishing.
|
|
76
|
+
DATASETS = {
|
|
77
|
+
"swe-bench-verified": {
|
|
78
|
+
"title": "SWE-bench Verified",
|
|
79
|
+
"what": (
|
|
80
|
+
"500 real GitHub issues drawn from 12 widely used Python projects — Django, "
|
|
81
|
+
"SymPy, scikit-learn, matplotlib, requests and others. Each task hands the agent "
|
|
82
|
+
"a repository at the commit before the fix, plus the issue text."
|
|
83
|
+
),
|
|
84
|
+
"scored": (
|
|
85
|
+
"The agent must produce a patch. It is applied to a clean checkout and the test "
|
|
86
|
+
"suite is run: FAIL_TO_PASS tests must flip from failing to passing, and "
|
|
87
|
+
"PASS_TO_PASS tests must stay green — so a patch cannot pass by breaking "
|
|
88
|
+
"unrelated behaviour."
|
|
89
|
+
),
|
|
90
|
+
"note": (
|
|
91
|
+
"The “Verified” subset was human-annotated to remove tasks that were unsolvable "
|
|
92
|
+
"or had broken tests, so a failure here reflects the agent, not the benchmark."
|
|
93
|
+
),
|
|
94
|
+
},
|
|
95
|
+
"swe-bench": {
|
|
96
|
+
"title": "SWE-bench",
|
|
97
|
+
"what": (
|
|
98
|
+
"Real GitHub issues from open-source Python projects, each paired with the "
|
|
99
|
+
"repository at the commit before the fix. The agent reads the issue and must "
|
|
100
|
+
"produce a working patch."
|
|
101
|
+
),
|
|
102
|
+
"scored": (
|
|
103
|
+
"The patch is applied to a clean checkout and the project's tests are run: "
|
|
104
|
+
"FAIL_TO_PASS tests must flip from failing to passing, and PASS_TO_PASS tests "
|
|
105
|
+
"must stay green."
|
|
106
|
+
),
|
|
107
|
+
"note": (
|
|
108
|
+
"Recorded simply as “swe-bench” by the harness, so the exact split "
|
|
109
|
+
"(full, Lite or Verified) isn't distinguished here."
|
|
110
|
+
),
|
|
111
|
+
},
|
|
112
|
+
"terminal-bench": {
|
|
113
|
+
"title": "Terminal-Bench",
|
|
114
|
+
"what": (
|
|
115
|
+
"Tasks that exercise a real terminal inside an isolated container — building and "
|
|
116
|
+
"configuring software, debugging a broken environment, driving command-line tools "
|
|
117
|
+
"end to end. From the same team as Harbor."
|
|
118
|
+
),
|
|
119
|
+
"scored": (
|
|
120
|
+
"Each task ships a test script that runs after the agent finishes. The task counts "
|
|
121
|
+
"as solved only if that script exits clean, so the agent is judged on the final "
|
|
122
|
+
"state of the machine rather than on what it said it did."
|
|
123
|
+
),
|
|
124
|
+
"note": "",
|
|
125
|
+
},
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def dataset_title(name):
|
|
130
|
+
"""Human name for a dataset slug, falling back to the slug itself."""
|
|
131
|
+
entry = DATASETS.get(name)
|
|
132
|
+
return entry["title"] if entry else name
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def dataset_info(name):
|
|
136
|
+
return DATASETS.get(name) or {
|
|
137
|
+
"title": name,
|
|
138
|
+
"what": "No description recorded for this dataset yet.",
|
|
139
|
+
"scored": "",
|
|
140
|
+
"note": "",
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# Runs-table columns, in render order. Drives the show/hide menu; the template
|
|
145
|
+
# tags every th/td with the matching data-col.
|
|
146
|
+
COLUMNS = [
|
|
147
|
+
("run", "Run"), ("agent", "Agent"), ("version", "Version"), ("start", "Start"), ("end", "End"), ("time", "Time"),
|
|
148
|
+
("model", "Model"), ("dataset", "Dataset"), ("mix", "Mix"),
|
|
149
|
+
("trials", "Trials"), ("solved", "Solved"), ("raw", "Raw"),
|
|
150
|
+
("adjusted", "Adjusted"), ("infra", "Infra errors"), ("broke", "Broke tests"),
|
|
151
|
+
("tokens", "Tokens"), ("cost", "Cost"),
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# Active roots, mutable at runtime via the workspace picker. Seeded at
|
|
156
|
+
# startup from CLI args / env; roots may be empty, in which case the UI asks.
|
|
157
|
+
# `label` is what the header shows — roots can be an internal detail (an upload
|
|
158
|
+
# scans a temp dir *and* its child, which is meaningless to read).
|
|
159
|
+
# The open workspace, and what it resolved to. `roots` and `label` are what
|
|
160
|
+
# the user pointed at; `job_roots` is after the foreign trees among them were
|
|
161
|
+
# converted — the directories parse actually walks. Conversion happens when
|
|
162
|
+
# the workspace is opened or refreshed, not on every request.
|
|
163
|
+
STATE: dict = {"roots": [], "label": "", "workspace": None, "job_roots": []}
|
|
164
|
+
WS_HOME = workspace.home()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# A path containing any of these is treated as a glob, so one entry can name
|
|
168
|
+
# several run directories: jobs_*, current/jobs_*/2026-08-*, **/jobs.
|
|
169
|
+
GLOB_CHARS = "*?["
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def is_pattern(value) -> bool:
|
|
173
|
+
return any(ch in (value or "") for ch in GLOB_CHARS)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def resolve_submitted_paths(values):
|
|
177
|
+
"""Split submitted paths into (existing dirs, unusable), preserving order.
|
|
178
|
+
|
|
179
|
+
Entries containing glob metacharacters expand to every directory they
|
|
180
|
+
match, so `.../jobs_*` picks up all of them in one field. Non-matching
|
|
181
|
+
patterns are reported like a missing directory rather than silently
|
|
182
|
+
resolving to nothing.
|
|
183
|
+
"""
|
|
184
|
+
good, bad = [], []
|
|
185
|
+
for raw in values:
|
|
186
|
+
raw = (raw or "").strip()
|
|
187
|
+
if not raw:
|
|
188
|
+
continue
|
|
189
|
+
resolved = os.path.abspath(os.path.expanduser(raw))
|
|
190
|
+
if is_pattern(raw):
|
|
191
|
+
matches = sorted(p for p in glob.glob(resolved, recursive=True) if os.path.isdir(p))
|
|
192
|
+
if not matches:
|
|
193
|
+
bad.append(raw)
|
|
194
|
+
for p in matches:
|
|
195
|
+
if p not in good:
|
|
196
|
+
good.append(p)
|
|
197
|
+
elif os.path.isdir(resolved):
|
|
198
|
+
if resolved not in good:
|
|
199
|
+
good.append(resolved)
|
|
200
|
+
else:
|
|
201
|
+
bad.append(raw)
|
|
202
|
+
return good, bad
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def suggest_directories(value):
|
|
206
|
+
"""Complete one path component, including linked and mounted directories."""
|
|
207
|
+
value = (value or "").lstrip()
|
|
208
|
+
if not value or len(value) > 4096 or "\0" in value or is_pattern(value):
|
|
209
|
+
return []
|
|
210
|
+
expanded = os.path.expanduser(value)
|
|
211
|
+
if value == "~":
|
|
212
|
+
expanded += os.sep
|
|
213
|
+
parent, prefix = os.path.split(expanded)
|
|
214
|
+
parent = os.path.abspath(parent or ".")
|
|
215
|
+
matches = []
|
|
216
|
+
try:
|
|
217
|
+
with os.scandir(parent) as entries:
|
|
218
|
+
# Suggestions never recurse or enumerate an unbounded directory.
|
|
219
|
+
for entry in islice(entries, 5000):
|
|
220
|
+
if not os.path.normcase(entry.name).startswith(os.path.normcase(prefix)):
|
|
221
|
+
continue
|
|
222
|
+
if entry.name.startswith(".") and not prefix.startswith("."):
|
|
223
|
+
continue
|
|
224
|
+
try:
|
|
225
|
+
if entry.is_dir():
|
|
226
|
+
matches.append(os.path.join(parent, entry.name) + os.sep)
|
|
227
|
+
except OSError:
|
|
228
|
+
continue
|
|
229
|
+
except OSError:
|
|
230
|
+
return []
|
|
231
|
+
return sorted(matches, key=os.path.normcase)[:20]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _bad_paths_message(bad):
|
|
235
|
+
"""Patterns that matched nothing read differently from a missing path."""
|
|
236
|
+
patterns = [p for p in bad if is_pattern(p)]
|
|
237
|
+
plain = [p for p in bad if not is_pattern(p)]
|
|
238
|
+
parts = []
|
|
239
|
+
if plain:
|
|
240
|
+
listed = ", ".join(f'<span class="mono">{escape(p)}</span>' for p in plain)
|
|
241
|
+
parts.append(f"No {'directory' if len(plain) == 1 else 'directories'} at {listed}")
|
|
242
|
+
if patterns:
|
|
243
|
+
listed = ", ".join(f'<span class="mono">{escape(p)}</span>' for p in patterns)
|
|
244
|
+
parts.append(f"Nothing matches {listed}")
|
|
245
|
+
return Markup(" · ".join(parts))
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def set_source(roots, label=None) -> None:
|
|
249
|
+
"""Point the app at *roots*, with *label* as the human-facing name."""
|
|
250
|
+
STATE["roots"] = roots
|
|
251
|
+
if label:
|
|
252
|
+
STATE["label"] = label
|
|
253
|
+
elif len(roots) == 1:
|
|
254
|
+
STATE["label"] = roots[0]
|
|
255
|
+
else:
|
|
256
|
+
STATE["label"] = f"{roots[0]} +{len(roots) - 1}" if roots else ""
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _import_roots(roots, import_root):
|
|
260
|
+
"""Convert any root a foreign harness recognises, returning job roots.
|
|
261
|
+
|
|
262
|
+
Which harness wrote a run is a property of the directory, not a question
|
|
263
|
+
for whoever opens it: the shapes are distinguishable, and someone handed a
|
|
264
|
+
results folder often cannot answer. Detection identifies it, and
|
|
265
|
+
a harness added to `nullius/importers/` is readable here without a UI
|
|
266
|
+
change — the point of the registry.
|
|
267
|
+
|
|
268
|
+
What one job is under a claimed root is the importer's answer
|
|
269
|
+
(`harnesses.trees`), matching the CLI's job boundaries. A root nothing
|
|
270
|
+
claims goes through as it is: a Harbor job directory or a parent of
|
|
271
|
+
them, which `find_job_dirs` walks.
|
|
272
|
+
|
|
273
|
+
A tree the importer claims but refuses is left out, so one bad tree
|
|
274
|
+
never costs the others or the process — but not silently: the reason,
|
|
275
|
+
redacted where it was raised, is flashed to the page being served, where
|
|
276
|
+
the picker shows it in place of "Nothing job-shaped", and logged when
|
|
277
|
+
there is no page (a workspace reopened at start-up).
|
|
278
|
+
"""
|
|
279
|
+
resolved, seen = [], set()
|
|
280
|
+
|
|
281
|
+
def take(path):
|
|
282
|
+
if path not in seen:
|
|
283
|
+
seen.add(path)
|
|
284
|
+
resolved.append(path)
|
|
285
|
+
|
|
286
|
+
for root in roots:
|
|
287
|
+
module = harnesses.detect(root) if os.path.isdir(root) else None
|
|
288
|
+
if module is None:
|
|
289
|
+
take(root)
|
|
290
|
+
continue
|
|
291
|
+
for tree in harnesses.trees(module, root):
|
|
292
|
+
try:
|
|
293
|
+
for job_dir in module.convert(tree, import_root):
|
|
294
|
+
take(job_dir)
|
|
295
|
+
except (OSError, SystemExit) as e:
|
|
296
|
+
why = redact(str(e) or f"{module.NAME} could not read it")
|
|
297
|
+
app.logger.warning("Not imported: %s — %s", redact(tree), why)
|
|
298
|
+
if has_request_context():
|
|
299
|
+
flash(Markup(f'Not imported: <span class="mono">{escape(redact(tree))}</span> '
|
|
300
|
+
f'— {escape(why)}'), "error")
|
|
301
|
+
return resolved
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def open_workspace(ws, refresh=False):
|
|
305
|
+
"""Make *ws* the one the app reads: convert its foreign roots once, then
|
|
306
|
+
every request walks the result and reads parsed jobs from its cache."""
|
|
307
|
+
STATE["workspace"] = ws
|
|
308
|
+
set_source(ws.roots, ws.label)
|
|
309
|
+
if refresh:
|
|
310
|
+
workspace.clear_cache(ws)
|
|
311
|
+
STATE["job_roots"] = _import_roots(ws.roots, ws.imports_dir)
|
|
312
|
+
ws.job_roots = list(STATE["job_roots"])
|
|
313
|
+
workspace.touch(ws)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def close_workspace():
|
|
317
|
+
STATE.update({"workspace": None, "roots": [], "label": "", "job_roots": []})
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def adopt_roots(roots, label="", new=False):
|
|
321
|
+
"""Roots the user just pointed at: into the open workspace, or a new one.
|
|
322
|
+
|
|
323
|
+
Without a label, roots that already have a workspace reopen it — so a
|
|
324
|
+
restart with the same argv lands in the same place, cache and all.
|
|
325
|
+
"""
|
|
326
|
+
ws = STATE["workspace"]
|
|
327
|
+
if ws and not new:
|
|
328
|
+
ws.roots = list(ws.roots) + [r for r in roots if r not in ws.roots]
|
|
329
|
+
workspace.save(ws)
|
|
330
|
+
open_workspace(ws)
|
|
331
|
+
return ws
|
|
332
|
+
ws = None if label else workspace.find_by_roots(WS_HOME, roots, besides=execution.runs_root())
|
|
333
|
+
ws = ws or workspace.create(WS_HOME, roots, label)
|
|
334
|
+
open_workspace(ws)
|
|
335
|
+
return ws
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _load_jobs():
|
|
339
|
+
ws = STATE["workspace"]
|
|
340
|
+
if ws is None:
|
|
341
|
+
return []
|
|
342
|
+
return workspace.load_jobs(ws, STATE["job_roots"])
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _jobs_for_workspace(name=None):
|
|
346
|
+
"""Read an indexed workspace without making it active or converting sources.
|
|
347
|
+
Once per request and workspace: a page that asks for the jobs several
|
|
348
|
+
times (the job page's controls, the rail, a summary) walks them once."""
|
|
349
|
+
memo = g.setdefault("_jobs", {}) if has_request_context() else {}
|
|
350
|
+
key = name if name and not (STATE["workspace"] and name == STATE["workspace"].name) else ""
|
|
351
|
+
if key in memo:
|
|
352
|
+
return memo[key]
|
|
353
|
+
if not key:
|
|
354
|
+
jobs = _load_jobs()
|
|
355
|
+
else:
|
|
356
|
+
ws = workspace.load(WS_HOME, name)
|
|
357
|
+
if ws is None:
|
|
358
|
+
abort(404)
|
|
359
|
+
jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots)
|
|
360
|
+
memo[key] = jobs
|
|
361
|
+
return jobs
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _record_workspace():
|
|
365
|
+
name = request.args.get("workspace") if has_request_context() else None
|
|
366
|
+
if not name:
|
|
367
|
+
return STATE["workspace"]
|
|
368
|
+
ws = workspace.load(WS_HOME, name)
|
|
369
|
+
if ws is None:
|
|
370
|
+
abort(404)
|
|
371
|
+
return ws
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _jobs_for_request():
|
|
375
|
+
return _jobs_for_workspace(request.args.get("workspace"))
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
@app.url_defaults
|
|
379
|
+
def _preserve_record_workspace(endpoint, values):
|
|
380
|
+
"""Carry evidence provenance through links, forms, and lazy detail requests."""
|
|
381
|
+
record_endpoints = {"job", "trial", "save_trace", "artifact_page", "conversation_data",
|
|
382
|
+
"trace_span_data"}
|
|
383
|
+
if has_request_context() and endpoint in record_endpoints and "workspace" not in values:
|
|
384
|
+
name = request.args.get("workspace")
|
|
385
|
+
if name:
|
|
386
|
+
values["workspace"] = name
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _find_job(jobs, job_id):
|
|
390
|
+
return next((j for j in jobs if j.job_id == job_id), None)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _safe_dest(base: str, rel: str) -> str | None:
|
|
394
|
+
"""Join *rel* onto *base*, refusing anything that escapes *base*."""
|
|
395
|
+
dest = os.path.normpath(os.path.join(base, rel))
|
|
396
|
+
if os.path.commonpath([base, dest]) != base:
|
|
397
|
+
return None
|
|
398
|
+
return dest
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
@app.template_global()
|
|
402
|
+
def view_query(filt, chart_filt):
|
|
403
|
+
"""Every argument a trial view is defined by, as one set of `url_for` kwargs.
|
|
404
|
+
|
|
405
|
+
The two filters answer different questions — `trace.Filter` prunes spans out
|
|
406
|
+
of the whole view, `charts.ChartFilter` only says which kinds one chart
|
|
407
|
+
draws — but a link has to carry both, or clicking either control silently
|
|
408
|
+
discards the other. Each still owns the naming of its own query keys.
|
|
409
|
+
"""
|
|
410
|
+
return {**filt.query, **chart_filt.query}
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
@app.template_global()
|
|
414
|
+
def task_labels_for(job):
|
|
415
|
+
"""Compact labels for this run only; IDs and ambiguous names stay intact."""
|
|
416
|
+
dataset = os.path.basename((job.dataset or "").rstrip("/")).split("@", 1)[0]
|
|
417
|
+
prefix = f"{dataset}__" if dataset and dataset != "mixed" else ""
|
|
418
|
+
labels, originals = {}, {}
|
|
419
|
+
for trial in job.trials:
|
|
420
|
+
full = trial.task_base
|
|
421
|
+
short = full[len(prefix):] if prefix and full.startswith(prefix) else full
|
|
422
|
+
short = short or full
|
|
423
|
+
labels[trial.trial_name] = short
|
|
424
|
+
originals.setdefault(short, set()).add(full)
|
|
425
|
+
return {trial.trial_name: (trial.task_base if len(originals[labels[trial.trial_name]]) > 1
|
|
426
|
+
else labels[trial.trial_name])
|
|
427
|
+
for trial in job.trials}
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
_CATALOG_CACHE: dict = {"signature": None, "cat": None}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _catalog_cached():
|
|
434
|
+
# read again when any of its files changes, or a benchmark folder comes or goes
|
|
435
|
+
signature = catalog.signature()
|
|
436
|
+
if all(mtime is None for _, mtime in signature):
|
|
437
|
+
return None
|
|
438
|
+
if _CATALOG_CACHE["signature"] != signature:
|
|
439
|
+
_CATALOG_CACHE.update(signature=signature, cat=catalog.load())
|
|
440
|
+
return _CATALOG_CACHE["cat"]
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _csrf_token() -> str:
|
|
444
|
+
token = session.get("csrf")
|
|
445
|
+
if not token:
|
|
446
|
+
token = secrets.token_urlsafe(32)
|
|
447
|
+
session["csrf"] = token
|
|
448
|
+
return token
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _origin_of(url: str) -> str:
|
|
452
|
+
"""scheme://host[:port] of an Origin or Referer, compared whole: a prefix
|
|
453
|
+
test would let http://127.0.0.1:50570 pass for :5057."""
|
|
454
|
+
parts = urlsplit(url)
|
|
455
|
+
return f"{parts.scheme}://{parts.netloc}".lower()
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
# Every page's forms send it: the workspace menu in base.html posts from anywhere.
|
|
459
|
+
app.jinja_env.globals["csrf_token"] = lambda: _csrf_token()
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _interface_id() -> str:
|
|
463
|
+
"""A digest of the templates, static files and web routes a page is made of
|
|
464
|
+
and reads. A page rendered under another digest runs code this process no
|
|
465
|
+
longer serves."""
|
|
466
|
+
digest = hashlib.sha256()
|
|
467
|
+
for root in (os.path.join(app.root_path, app.template_folder), app.static_folder, os.path.join(app.root_path, "web")):
|
|
468
|
+
for folder, subfolders, names in os.walk(root):
|
|
469
|
+
subfolders[:] = sorted(d for d in subfolders if d != "__pycache__") # written at import, not code
|
|
470
|
+
for name in sorted(names):
|
|
471
|
+
path = os.path.join(folder, name)
|
|
472
|
+
digest.update(os.path.relpath(path, root).encode() + b"\0")
|
|
473
|
+
with open(path, "rb") as f:
|
|
474
|
+
digest.update(f.read())
|
|
475
|
+
return digest.hexdigest()[:16]
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
INTERFACE = _interface_id()
|
|
479
|
+
app.jinja_env.globals["interface_id"] = lambda: INTERFACE
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
@app.after_request
|
|
483
|
+
def _send_interface(response):
|
|
484
|
+
response.headers["X-Nullius-Interface"] = INTERFACE
|
|
485
|
+
return response
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
class StaleFormToken(Forbidden):
|
|
489
|
+
"""A token the session does not hold, as from a page loaded before a restart.
|
|
490
|
+
Its header tells the page to ask for a reload instead of showing the error."""
|
|
491
|
+
description = "Stale form token; reload the page and try again."
|
|
492
|
+
|
|
493
|
+
def get_headers(self, environ=None, scope=None):
|
|
494
|
+
return [*super().get_headers(environ, scope), ("X-Nullius-Reason", "stale-form-token")]
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _require_csrf():
|
|
498
|
+
"""The one guard every mutation of the environments takes: the session's
|
|
499
|
+
token, sent back by the form, and a same-origin request."""
|
|
500
|
+
sent = request.form.get("csrf") or request.headers.get("X-CSRF-Token")
|
|
501
|
+
if not sent:
|
|
502
|
+
abort(403, description="Missing form token; reload the page and try again.")
|
|
503
|
+
if not secrets.compare_digest(sent, session.get("csrf", "")):
|
|
504
|
+
raise StaleFormToken()
|
|
505
|
+
origin = request.headers.get("Origin") or request.headers.get("Referer")
|
|
506
|
+
if origin and _origin_of(origin) != request.host_url.rstrip("/").lower():
|
|
507
|
+
abort(403, description="Cross-origin requests cannot change environments.")
|