nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Read-only launch vocabulary for Harbor recipes; no harness imports.
|
|
2
|
+
|
|
3
|
+
The catalogue owns benchmark facts. This module owns only launch choices and
|
|
4
|
+
parameter metadata: the agents the builder offers by name and the parameters
|
|
5
|
+
it knows for them. The browser compiles recipes locally; this module never
|
|
6
|
+
executes commands, and nothing here imports a harness or a runtime.
|
|
7
|
+
|
|
8
|
+
Every name here has to match what Harbor accepts - an agent's import path, the
|
|
9
|
+
``--ak`` a parameter is sent as - and this repository has no Harbor source to
|
|
10
|
+
check them against. What is checked here is the other half:
|
|
11
|
+
``app/ui_tests/test_command_builder.py`` pins the recipes this module
|
|
12
|
+
produces, so a change made *here* cannot silently alter what the builder emits.
|
|
13
|
+
"""
|
|
14
|
+
from dataclasses import asdict
|
|
15
|
+
|
|
16
|
+
# `kind` groups the builder's agent list: the reference agent a benchmark
|
|
17
|
+
# brings and the controls that bracket it. Harbor's own agents join them once
|
|
18
|
+
# the pinned Harbor is scanned (`builtin`); an agent under evaluation is
|
|
19
|
+
# typed per recipe, as an import path.
|
|
20
|
+
AGENTS = [
|
|
21
|
+
# A benchmark's own reference agent: lives in Harbor's source tree, not its package,
|
|
22
|
+
# so it needs the adapter directory on PYTHONPATH (see running.md). The path is
|
|
23
|
+
# anchored on the run target's home, not on a sibling of a checkout: a run from
|
|
24
|
+
# the builder starts in its own run directory, on this machine or a host, and
|
|
25
|
+
# `../harbor-src` from there pointed nowhere. A recipe may name another.
|
|
26
|
+
{"id": "tau3-llm-agent", "label": "tau3-llm-agent (τ³ reference)", "arg": "tau3_llm_agent:Tau3LLMAgent",
|
|
27
|
+
"control": False, "benchmarks": ["tau3-bench"], "pythonpath": "~/harbor-src/adapters/tau3-bench", "kind": "reference",
|
|
28
|
+
"summary": "τ³'s own reference agent, from Harbor's source tree."},
|
|
29
|
+
{"id": "oracle", "label": "oracle", "arg": "oracle", "control": True, "kind": "control",
|
|
30
|
+
"summary": "Applies the task's reference solution; never calls a model. A ceiling, never ranked."},
|
|
31
|
+
{"id": "nop", "label": "nop", "arg": "nop", "control": True, "kind": "control",
|
|
32
|
+
"summary": "Does nothing. The floor a verifier must reject."},
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parameter(key, label, group, kind, default=None, *, help="", **extra):
|
|
37
|
+
return dict(key=key, label=label, group=group, type=kind, default=default, help=help, **extra)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
PARAMETERS = [
|
|
41
|
+
# Sent to the agent its group names: the τ³ reference agent's tool timeout,
|
|
42
|
+
# emitted as `--ak tool_timeout_sec=`.
|
|
43
|
+
parameter("mcp_timeout_s", "Tool call timeout · seconds", "tau3-llm-agent", "integer", 300, min=0,
|
|
44
|
+
help="How long one MCP tool call may take."),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# Local launch paths are examples from this repository, not claims about disk state.
|
|
49
|
+
LAUNCH = {
|
|
50
|
+
"swe-bench-verified": {"local": "../harbor-datasets/swe-bench-verified"},
|
|
51
|
+
"terminal-bench": {"local": "../harbor-datasets/terminal-bench"},
|
|
52
|
+
# A generator: the program `uv run` starts in a checkout of the repository that holds it, writing to --output-dir.
|
|
53
|
+
"tau3-bench": {"local": "../harbor-datasets/tau3-bench",
|
|
54
|
+
"generate": {"program": "tau3-bench", "checkout": "tau2-bench checkout · dev/tau3 branch", "example": "../tau2-bench"}},
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def payload(cat, jobs, workspace_name=""):
|
|
59
|
+
from nullius import providers
|
|
60
|
+
from nullius.standard import recipe as standard
|
|
61
|
+
models = sorted({j.model_name for j in jobs if j.model_name})
|
|
62
|
+
dated = [j for j in jobs if j.model_name and j.started_at]
|
|
63
|
+
latest = max(dated, key=lambda j: j.started_at.timestamp()).model_name if dated else ""
|
|
64
|
+
return {
|
|
65
|
+
"version": 1, "workspace": workspace_name or "unassigned", "agents": AGENTS,
|
|
66
|
+
# Harbor and the Nullius standard: the page offers choices, the core and standard/recipe.py compile them.
|
|
67
|
+
"runners": [standard.RUNNER, providers.HARBOR],
|
|
68
|
+
# why a saved recipe of a runner no longer offered does not open (a benchmark's own scripts: its Original setup)
|
|
69
|
+
"runnerGone": providers.GONE,
|
|
70
|
+
# the standard's benchmarks, agents and capabilities, and why a pair does not fit (standard/recipe.py, choices)
|
|
71
|
+
"standard": standard.choices(),
|
|
72
|
+
"parameters": PARAMETERS, "benchmarks": [asdict(b) | {"label": b.label} for b in cat.benchmarks],
|
|
73
|
+
"datasets": [asdict(d) | LAUNCH.get(d.key, {}) for d in cat.datasets],
|
|
74
|
+
"models": models, "defaultModel": latest,
|
|
75
|
+
"seenLocal": sorted({j.dataset_path for j in jobs if j.dataset_path}),
|
|
76
|
+
"seenRegistry": sorted({j.dataset for j in jobs if "@" in j.dataset}),
|
|
77
|
+
}
|
|
78
|
+
|
nullius/redact.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Secret redaction.
|
|
2
|
+
|
|
3
|
+
Job directories are arbitrary text from wherever an exception was raised, and
|
|
4
|
+
this app renders that text into a browser. Never surface raw text from a job
|
|
5
|
+
directory without passing it through :func:`redact` first.
|
|
6
|
+
|
|
7
|
+
``app/tests/test_redact.py`` pins what the patterns below catch and what they
|
|
8
|
+
leave alone.
|
|
9
|
+
|
|
10
|
+
Two properties matter:
|
|
11
|
+
|
|
12
|
+
* **Anchored.** ``sk-`` must start at a token boundary, or the pattern fires
|
|
13
|
+
inside base64 payloads and eats the rest of a diff.
|
|
14
|
+
* **Separator-agnostic.** Credentials arrive as ``KEY=v`` from shell commands
|
|
15
|
+
*and* as ``"key": "v"`` from JSON — ``trajectory.py`` renders tool arguments
|
|
16
|
+
with ``json.dumps``, so the JSON form is the common one here.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
|
|
23
|
+
_PATTERNS = (
|
|
24
|
+
# OpenAI-style keys. The lookbehind is the whole point: without it this
|
|
25
|
+
# matches "sk-" appearing mid-base64 and swallows everything after it.
|
|
26
|
+
re.compile(r"(?<![A-Za-z0-9_-])sk-[A-Za-z0-9_-]{8,}"),
|
|
27
|
+
# Bearer tokens in headers. Keeps the word "bearer" so the line still reads.
|
|
28
|
+
re.compile(r"(?i)\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}"),
|
|
29
|
+
# KEY=value and "key": "value", quoted or not, escaped or not. Longer
|
|
30
|
+
# alternatives lead: `access_token` must win before `token` can match it.
|
|
31
|
+
# The name may carry a prefix joined by `_` or `-` (GH_TOKEN,
|
|
32
|
+
# CURSOR_API_KEY, AWS_SECRET_ACCESS_KEY), which `\b` would refuse since `_`
|
|
33
|
+
# is a word character; a letter or digit before it still does not count.
|
|
34
|
+
re.compile(
|
|
35
|
+
r"(?i)(?<![A-Za-z0-9])(api[_-]?key|access[_-]?token|access[_-]?key|authorization|password|secret|token)"
|
|
36
|
+
r"(\s*[=:]\s*|\\?[\"']?\s*[=:]\s*\\?[\"']?)"
|
|
37
|
+
r"[A-Za-z0-9._~+/=-]{8,}"
|
|
38
|
+
),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# Credentials that need no name beside them: their prefix says what they are
|
|
42
|
+
# (GitHub, Hugging Face, AWS access key ids, Google API keys, Slack).
|
|
43
|
+
_BARE = re.compile(
|
|
44
|
+
r"(?<![A-Za-z0-9_-])(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|hf_[A-Za-z0-9]{20,}"
|
|
45
|
+
r"|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{30,}|xox[abeoprs]-[A-Za-z0-9-]{10,})"
|
|
46
|
+
)
|
|
47
|
+
# "Authorization: token X" and "Authorization: Basic X": the scheme word is
|
|
48
|
+
# kept, as with Bearer. Only after the header name, where "basic" is no word.
|
|
49
|
+
_SCHEME = re.compile(r"(?i)(authorization\\?[\"']?\s*[=:]\s*\\?[\"']?\s*(?:token|basic)\s+)[A-Za-z0-9._~+/=-]{8,}")
|
|
50
|
+
# user:password@ in a URL: the user stays, the password goes.
|
|
51
|
+
_USERINFO = re.compile(r"(?<=://)([^/\s:@\[\]]+):([^/\s@\[\]]+)@")
|
|
52
|
+
|
|
53
|
+
# Angle brackets would survive Jinja's autoescaping as <…>. Square
|
|
54
|
+
# brackets read the same in a browser as they do in a terminal.
|
|
55
|
+
_REPLACEMENT = "[REDACTED]"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def redact(text: str | None) -> str:
|
|
59
|
+
"""Return *text* with anything credential-shaped replaced.
|
|
60
|
+
|
|
61
|
+
The key name survives; only the value goes. An error message that reads
|
|
62
|
+
"auth failed for [REDACTED]" is still diagnostic, and over-redaction
|
|
63
|
+
destroys exactly the text someone opened the page to read.
|
|
64
|
+
"""
|
|
65
|
+
if not text:
|
|
66
|
+
return ""
|
|
67
|
+
out = _PATTERNS[0].sub(_REPLACEMENT, text)
|
|
68
|
+
out = _BARE.sub(_REPLACEMENT, out)
|
|
69
|
+
out = _USERINFO.sub(lambda m: m.group(0) if m.group(2) == _REPLACEMENT else f"{m.group(1)}:{_REPLACEMENT}@", out)
|
|
70
|
+
out = _SCHEME.sub(rf"\1{_REPLACEMENT}", out)
|
|
71
|
+
out = _PATTERNS[1].sub(rf"\1 {_REPLACEMENT}", out)
|
|
72
|
+
return _PATTERNS[2].sub(rf"\1\2{_REPLACEMENT}", out)
|
nullius/runs/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Bounded, redacted views of named trial records. No writes or arbitrary paths."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
from nullius.redact import redact
|
|
7
|
+
|
|
8
|
+
FILES = {
|
|
9
|
+
"result": ("Result record", "result.json"),
|
|
10
|
+
"config": ("Run configuration", "config.json"),
|
|
11
|
+
"exception": ("Exception", "exception.txt"),
|
|
12
|
+
"trial-log": ("Trial log", "trial.log"),
|
|
13
|
+
"gateway": ("Gateway log", "agent/gateway.log"),
|
|
14
|
+
"reward": ("Recorded reward", "verifier/reward.txt"),
|
|
15
|
+
"reward-json": ("Recorded reward (rewardkit)", "verifier/reward.json"),
|
|
16
|
+
"reward-details": ("Reward breakdown (rewardkit)", "verifier/reward-details.json"),
|
|
17
|
+
# Harbor's record of what it copied out of the container after the run —
|
|
18
|
+
# one entry per requested path with its status (`ok`, `failed`, `empty`,
|
|
19
|
+
# `skipped` — Harbor's `ArtifactManifestEntry`). The last of the nine
|
|
20
|
+
# files a trial writes to reach a page: the others were already
|
|
21
|
+
# here, or are read at parse.
|
|
22
|
+
"manifest": ("Artifact manifest", "artifacts/manifest.json"),
|
|
23
|
+
}
|
|
24
|
+
CHUNK_SIZE = 8_000
|
|
25
|
+
MAX_READ = 16 * 1024 * 1024
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def path_for(trial_dir, key):
|
|
29
|
+
if key not in FILES:
|
|
30
|
+
raise LookupError(key)
|
|
31
|
+
base = os.path.realpath(trial_dir)
|
|
32
|
+
path = os.path.realpath(os.path.join(base, FILES[key][1]))
|
|
33
|
+
if os.path.commonpath([base, path]) != base:
|
|
34
|
+
raise LookupError(key)
|
|
35
|
+
return path
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def listing(trial_dir):
|
|
39
|
+
records = []
|
|
40
|
+
for key, (label, filename) in FILES.items():
|
|
41
|
+
try:
|
|
42
|
+
path = path_for(trial_dir, key)
|
|
43
|
+
exists = os.path.isfile(path)
|
|
44
|
+
size = os.path.getsize(path) if exists else None
|
|
45
|
+
except (OSError, LookupError):
|
|
46
|
+
exists, size = False, None
|
|
47
|
+
records.append(dict(key=key, label=label, filename=filename, exists=exists, size=size))
|
|
48
|
+
return records
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read(trial_dir, key, page=1):
|
|
52
|
+
path = path_for(trial_dir, key)
|
|
53
|
+
with open(path, encoding="utf-8", errors="replace") as handle:
|
|
54
|
+
raw = handle.read(MAX_READ + 1)
|
|
55
|
+
# Redact before slicing so credentials cannot straddle visible chunks.
|
|
56
|
+
text = redact(raw[:MAX_READ])
|
|
57
|
+
pages = max(1, (len(text) + CHUNK_SIZE - 1) // CHUNK_SIZE)
|
|
58
|
+
try:
|
|
59
|
+
page = int(page)
|
|
60
|
+
except (TypeError, ValueError):
|
|
61
|
+
page = 1
|
|
62
|
+
page = min(pages, max(1, page))
|
|
63
|
+
start = (page - 1) * CHUNK_SIZE
|
|
64
|
+
return dict(key=key, label=FILES[key][0], filename=FILES[key][1], page=page,
|
|
65
|
+
pages=pages, text=text[start:start + CHUNK_SIZE], empty=not text,
|
|
66
|
+
truncated=len(raw) > MAX_READ, characters=len(text), start=start,
|
|
67
|
+
end=min(start + CHUNK_SIZE, len(text)))
|
nullius/runs/classify.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Outcome classification.
|
|
2
|
+
|
|
3
|
+
The whole point of the app: separate *infrastructure* failures (rate limits,
|
|
4
|
+
setup crashes) from genuine *agent* outcomes (solved / wrong patch / no patch),
|
|
5
|
+
so the solve rate reflects the agent and not the harness.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
# Outcome labels, ordered from best to worst for display.
|
|
11
|
+
RESOLVED = "resolved"
|
|
12
|
+
# A benchmark that gives a trial a score and no pass or fail (a token F1 over a
|
|
13
|
+
# conversation's questions): the trial was answered and measured, neither
|
|
14
|
+
# resolved nor failed. Its score is the measure; the run has no solve rate.
|
|
15
|
+
SCORED = "scored"
|
|
16
|
+
PATCH_FAILED_TESTS = "patch_failed_tests"
|
|
17
|
+
# The benchmark-neutral failure: the verifier said no, and the benchmark
|
|
18
|
+
# reports nothing finer than the score. What a failed trial of a dataset
|
|
19
|
+
# whose verifier reports no patch detail, or that the app knows only by its
|
|
20
|
+
# reward, classifies as.
|
|
21
|
+
FAILED_VERIFICATION = "failed_verification"
|
|
22
|
+
PATCH_NOT_APPLIED = "patch_not_applied"
|
|
23
|
+
NO_PATCH = "no_patch"
|
|
24
|
+
RATE_LIMITED = "rate_limited"
|
|
25
|
+
API_ERROR = "api_error"
|
|
26
|
+
SETUP_ERROR = "setup_error"
|
|
27
|
+
# The run's own spending cap refused a model call the unit needed (the
|
|
28
|
+
# standard's relay, ``standard.ledger``): the unit was cut off. Nothing
|
|
29
|
+
# failed, and no attempt was had: the money ran out, by the user's own word.
|
|
30
|
+
SPENDING_CAP = "spending_cap"
|
|
31
|
+
CANCELLED = "cancelled"
|
|
32
|
+
AGENT_ERROR = "agent_error"
|
|
33
|
+
UNKNOWN = "unknown"
|
|
34
|
+
|
|
35
|
+
# Outcomes excluded from the adjusted rate: infrastructure failures or
|
|
36
|
+
# interruptions imposed by the run, not the agent's ability.
|
|
37
|
+
INFRA_OUTCOMES = {RATE_LIMITED, API_ERROR, SETUP_ERROR, SPENDING_CAP, CANCELLED}
|
|
38
|
+
|
|
39
|
+
ORDER = [
|
|
40
|
+
RESOLVED,
|
|
41
|
+
SCORED,
|
|
42
|
+
PATCH_FAILED_TESTS,
|
|
43
|
+
FAILED_VERIFICATION,
|
|
44
|
+
PATCH_NOT_APPLIED,
|
|
45
|
+
NO_PATCH,
|
|
46
|
+
AGENT_ERROR,
|
|
47
|
+
RATE_LIMITED,
|
|
48
|
+
API_ERROR,
|
|
49
|
+
SETUP_ERROR,
|
|
50
|
+
CANCELLED,
|
|
51
|
+
SPENDING_CAP,
|
|
52
|
+
UNKNOWN,
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
# Harbor names its provider failures Api*, all descending from ApiError, so the
|
|
56
|
+
# prefix follows their hierarchy instead of guessing at it — and a class they
|
|
57
|
+
# add later is classified correctly without us noticing it exists.
|
|
58
|
+
#
|
|
59
|
+
# The two below are the exceptions to that rule: they are provider or config
|
|
60
|
+
# failures that don't carry the prefix. Everything Harbor ships is pinned by a
|
|
61
|
+
# test, so this list cannot quietly fall behind theirs.
|
|
62
|
+
_API_PREFIX = "api"
|
|
63
|
+
_INFRA_EXCEPTIONS = {
|
|
64
|
+
"agentauthenticationerror", # bad or missing credentials — config, not skill
|
|
65
|
+
"modelnotfounderror", # the model name is wrong; the agent never ran
|
|
66
|
+
"networkconnectionerror", # DNS, refused connection, TLS — transport, not skill
|
|
67
|
+
"unknownapierror", # Harbor's own "some provider error"; an ApiError without the prefix
|
|
68
|
+
}
|
|
69
|
+
# What a unit the run's spending cap cut off carries (``standard.ledger.ERROR``): recorded by the relay that
|
|
70
|
+
# refused the call, never inferred.
|
|
71
|
+
_CAP_EXCEPTION = "spendingcaperror"
|
|
72
|
+
|
|
73
|
+
# What Harbor's agents raise when the agent itself fails — its budget spent,
|
|
74
|
+
# its own context filled, its own output truncated, a refusal, its process
|
|
75
|
+
# exiting non-zero. Each is the agent's behaviour under conditions it was
|
|
76
|
+
# given, so each is a result and belongs in the denominator, charged to the
|
|
77
|
+
# agent. Deliberately *not* infra; move one only with a reason written down.
|
|
78
|
+
#
|
|
79
|
+
# The list is explicit and short on purpose. An exception on none of these
|
|
80
|
+
# lists is UNKNOWN: still in the denominator, charged to nobody. The
|
|
81
|
+
# alternative — everything unmatched is the agent's fault — made a crash in
|
|
82
|
+
# the harness, the verifier or this reader read as poor coding. The trial
|
|
83
|
+
# page shows the recorded type beside the label, so a reader who meets a new
|
|
84
|
+
# one can place it here.
|
|
85
|
+
_AGENT_EXCEPTIONS = {
|
|
86
|
+
"agenttimeouterror", # the agent spent its budget — that is a result
|
|
87
|
+
"unitspendingcaperror", # the unit spent its own spending cap (the standard's relay): the same, in money
|
|
88
|
+
"agentsafetyrefusalerror", # the agent (or its provider) chose to refuse
|
|
89
|
+
"contextwindowexceedederror", # the agent filled its own context
|
|
90
|
+
"contextlengthexceedederror", # same, from Harbor's LLM layer
|
|
91
|
+
"outputlengthexceedederror", # the agent's reply hit max_tokens
|
|
92
|
+
"outputtokenexceedederror", # same, seen from an installed agent's output
|
|
93
|
+
"nonzeroagentexitcodeerror", # the agent's process died and Harbor found no finer reason
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
LABELS = {
|
|
97
|
+
RESOLVED: "Resolved",
|
|
98
|
+
SCORED: "Scored",
|
|
99
|
+
PATCH_FAILED_TESTS: "Patch failed tests",
|
|
100
|
+
FAILED_VERIFICATION: "Failed verification",
|
|
101
|
+
PATCH_NOT_APPLIED: "Patch did not apply",
|
|
102
|
+
NO_PATCH: "No patch produced",
|
|
103
|
+
AGENT_ERROR: "Agent error",
|
|
104
|
+
RATE_LIMITED: "Rate limited",
|
|
105
|
+
API_ERROR: "API error",
|
|
106
|
+
SETUP_ERROR: "Setup error",
|
|
107
|
+
SPENDING_CAP: "Spending cap",
|
|
108
|
+
CANCELLED: "Run stopped",
|
|
109
|
+
UNKNOWN: "Unknown",
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# One-line meaning of each outcome, for the glossary the UI renders. Lives here
|
|
114
|
+
# rather than in a template so the wording follows the classifier that produces
|
|
115
|
+
# it — change the rule below, change the sentence with it.
|
|
116
|
+
DESCRIPTIONS = {
|
|
117
|
+
RESOLVED: "The patch applied and the verifier's tests all passed. A genuine solve.",
|
|
118
|
+
SCORED: (
|
|
119
|
+
"The benchmark gave the trial a score and no pass or fail - a token F1 over a "
|
|
120
|
+
"conversation's questions, partial credit. Answered and measured, neither "
|
|
121
|
+
"solved nor failed: the score is the result, and the run has no solve rate."
|
|
122
|
+
),
|
|
123
|
+
PATCH_FAILED_TESTS: (
|
|
124
|
+
"The agent produced a patch that applied cleanly, but the tests still failed. "
|
|
125
|
+
"A real attempt that got the answer wrong — the most informative failure."
|
|
126
|
+
),
|
|
127
|
+
FAILED_VERIFICATION: (
|
|
128
|
+
"The verifier ran and scored the attempt below success, on a benchmark that "
|
|
129
|
+
"reports no patch detail, or that the app knows only "
|
|
130
|
+
"by its reward. A real attempt that didn't pass; counted against the agent."
|
|
131
|
+
),
|
|
132
|
+
PATCH_NOT_APPLIED: (
|
|
133
|
+
"A patch was produced but wouldn't apply to the repository — usually malformed "
|
|
134
|
+
"context lines or edits against the wrong revision."
|
|
135
|
+
),
|
|
136
|
+
NO_PATCH: "The agent finished without producing a patch at all.",
|
|
137
|
+
AGENT_ERROR: (
|
|
138
|
+
"The agent itself failed — it timed out, filled its context, truncated its "
|
|
139
|
+
"own output, refused, spent its own spending cap, or its process exited non-zero. Counted against the "
|
|
140
|
+
"agent, because each of those is a result rather than an interruption."
|
|
141
|
+
),
|
|
142
|
+
RATE_LIMITED: (
|
|
143
|
+
"The model API returned a rate-limit error (HTTP 429) and the run died. "
|
|
144
|
+
"Infrastructure, not capability — excluded from the adjusted rate."
|
|
145
|
+
),
|
|
146
|
+
API_ERROR: (
|
|
147
|
+
"The model API failed some other way — overloaded, unreachable, out of "
|
|
148
|
+
"quota, or the key was rejected. The agent never got a fair attempt, so "
|
|
149
|
+
"this is infrastructure too and is excluded from the adjusted rate."
|
|
150
|
+
),
|
|
151
|
+
SETUP_ERROR: (
|
|
152
|
+
"The environment or image failed to build or start, so the agent never ran. "
|
|
153
|
+
"Infrastructure, not capability — excluded from the adjusted rate."
|
|
154
|
+
),
|
|
155
|
+
SPENDING_CAP: (
|
|
156
|
+
"The run reached its spending cap: its relay refused a model call this "
|
|
157
|
+
"trial needed, so the trial was cut off. Nothing failed and no attempt "
|
|
158
|
+
"was had, so it is excluded from the adjusted rate."
|
|
159
|
+
),
|
|
160
|
+
CANCELLED: (
|
|
161
|
+
"The run was stopped while this trial was in progress. Its attempt was interrupted, "
|
|
162
|
+
"so it is excluded from the adjusted rate."
|
|
163
|
+
),
|
|
164
|
+
UNKNOWN: (
|
|
165
|
+
"No verifier report and no exception, or an exception this reader does not "
|
|
166
|
+
"recognise — a crash in the harness, the verifier or the reader itself. The "
|
|
167
|
+
"outcome could not be determined; it stays in the denominator, charged to "
|
|
168
|
+
"nobody, and the recorded type is shown so the classifier can be extended."
|
|
169
|
+
),
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def from_verdict(exception_type, verdict):
|
|
174
|
+
"""Return an outcome label from the exception type and a `Verdict`.
|
|
175
|
+
|
|
176
|
+
An exception outranks the verifier: a trial that crashed did not earn
|
|
177
|
+
whatever the verifier managed to say about it. Which side it is charged
|
|
178
|
+
to comes from the lists above; an exception on none of them is `UNKNOWN`.
|
|
179
|
+
|
|
180
|
+
The patch-shaped labels are reachable only for a benchmark whose reader
|
|
181
|
+
fills `produced` / `usable`. A verdict that failed without that detail is
|
|
182
|
+
`FAILED_VERIFICATION` — the verifier said no, and this benchmark has
|
|
183
|
+
nothing further to say about why — rather than borrowing a label about
|
|
184
|
+
patches that would not be true.
|
|
185
|
+
"""
|
|
186
|
+
if exception_type:
|
|
187
|
+
et = exception_type.lower().rpartition(".")[2] # tolerate dotted paths
|
|
188
|
+
if et == "cancellederror":
|
|
189
|
+
return CANCELLED
|
|
190
|
+
if et == _CAP_EXCEPTION:
|
|
191
|
+
return SPENDING_CAP
|
|
192
|
+
# Rate limiting first: it is an Api* class too, but common enough to be
|
|
193
|
+
# worth seeing on its own rather than folded into the generic bucket.
|
|
194
|
+
if "ratelimit" in et or "rate_limit" in et or "429" in et:
|
|
195
|
+
return RATE_LIMITED
|
|
196
|
+
if "setup" in et or "environment" in et or "build" in et:
|
|
197
|
+
return SETUP_ERROR
|
|
198
|
+
if et.startswith(_API_PREFIX) or et in _INFRA_EXCEPTIONS:
|
|
199
|
+
return API_ERROR
|
|
200
|
+
if et in _AGENT_EXCEPTIONS:
|
|
201
|
+
return AGENT_ERROR
|
|
202
|
+
# Neither side claims it. Not the agent's fault by default: the
|
|
203
|
+
# trial stays counted and nobody is charged.
|
|
204
|
+
return UNKNOWN
|
|
205
|
+
|
|
206
|
+
if verdict is None or verdict.succeeded is None:
|
|
207
|
+
return UNKNOWN
|
|
208
|
+
if verdict.succeeded:
|
|
209
|
+
return RESOLVED
|
|
210
|
+
if verdict.produced is None:
|
|
211
|
+
return FAILED_VERIFICATION
|
|
212
|
+
if not verdict.produced:
|
|
213
|
+
return NO_PATCH
|
|
214
|
+
if verdict.usable is False:
|
|
215
|
+
return PATCH_NOT_APPLIED
|
|
216
|
+
return PATCH_FAILED_TESTS
|