nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/judge/state.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""The text a question reads, built from the redacted views only: the
|
|
2
|
+
instruction, the agent's steps (messages and tool calls), the final message,
|
|
3
|
+
the recorded error, the verifier text. Truncated head-and-tail to a stated
|
|
4
|
+
size with a marker, never silently. Nothing raw leaves this module."""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
from nullius.redact import redact
|
|
12
|
+
|
|
13
|
+
LIMIT = 60_000 # characters of state per call; the judge is priced per input token
|
|
14
|
+
OBSERVATION = 600 # characters kept of a tool observation: the head, and the tail below
|
|
15
|
+
OBSERVATION_TAIL = 200 # of those, the characters kept from the end - where a traceback names its exception and a command its exit
|
|
16
|
+
MARKER = "\n[… {n} characters left out here …]\n"
|
|
17
|
+
|
|
18
|
+
# Observation relevance. Accuracy falls as state grows with material unrelated
|
|
19
|
+
# to the decision, so a tool observation is sent in full (up to OBSERVATION)
|
|
20
|
+
# only when it bears on the question - deterministically, in code, by the
|
|
21
|
+
# rules in `_relevant` - and is otherwise a one-line stub with a visible
|
|
22
|
+
# marker. The instruction, the agent's messages, the tool names and their
|
|
23
|
+
# arguments are never dropped; only what a tool answered is. `relevance="all"`
|
|
24
|
+
# opts out.
|
|
25
|
+
RELEVANCE = "question"
|
|
26
|
+
OMITTED = "[observation omitted: {n} characters, not relevant to the question]"
|
|
27
|
+
KEPT_MARKER = "[observations: {kept} kept, {omitted} omitted as not relevant to the question]"
|
|
28
|
+
MIN_TERM = 5 # characters a word needs to count as a term of the instruction
|
|
29
|
+
MIN_OVERLAP = 2 # instruction terms an observation must share to be kept for overlap alone
|
|
30
|
+
_ERRORISH = re.compile(r"(?i)\b(error|errno|exception|traceback|fail(?:ed|ure|s)?|denied|not found|no such file|fatal|panic|refused|timed? ?out|cannot|can't)\b")
|
|
31
|
+
_WORD = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*")
|
|
32
|
+
|
|
33
|
+
# A state kind a later question group adds: `register(kind, builder)`, the
|
|
34
|
+
# builder taking the same keywords `build` does and returning the *raw* text;
|
|
35
|
+
# `build` redacts and truncates it, so nothing raw leaves here either way.
|
|
36
|
+
BUILDERS: dict = {}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def register(kind: str, builder) -> None:
|
|
40
|
+
"""Declare how a state kind is built. `builder(traj, trial=, trial_dir=, other=, **extra) -> str`."""
|
|
41
|
+
if kind in BUILDERS or kind in ("trial", "failure", "final", "pair"):
|
|
42
|
+
raise ValueError(f"state kind {kind!r} is already declared")
|
|
43
|
+
BUILDERS[kind] = builder
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def truncate(text: str, limit: int = LIMIT) -> tuple:
|
|
47
|
+
"""(text, truncated): the head and the tail with a marker between when
|
|
48
|
+
the text is longer than the limit."""
|
|
49
|
+
if len(text) <= limit:
|
|
50
|
+
return text, False
|
|
51
|
+
left = len(text) - limit
|
|
52
|
+
head = limit * 2 // 3
|
|
53
|
+
return text[:head] + MARKER.format(n=left) + text[-(limit - head):], True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def instruction_of(traj) -> str:
|
|
57
|
+
"""The task as the agent was given it: the first user step. Many Harbor
|
|
58
|
+
agents (mini-swe-agent, swe-agent, OpenHands, Codex) open with their own
|
|
59
|
+
system prompt, which is not the task; only a trajectory with no user step
|
|
60
|
+
falls back to its first step that is not the agent's."""
|
|
61
|
+
first = next((s.message for s in traj.steps if s.source == "user" and s.message), "")
|
|
62
|
+
return first or next((s.message for s in traj.steps if s.source not in ("agent", "system") and s.message), "") \
|
|
63
|
+
or next((s.message for s in traj.steps if s.source != "agent" and s.message), "")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _terms(text: str) -> set:
|
|
67
|
+
return {w.lower() for w in _WORD.findall(text or "") if len(w) >= MIN_TERM}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _relevant(call: dict, obs: str, instruction: str, terms: set, next_message: str) -> bool:
|
|
71
|
+
"""Whether an observation bears on the question, by rule: it reports a
|
|
72
|
+
failure; its call edited or executed something; its target is named by
|
|
73
|
+
the instruction or by the agent's next message; or it shares MIN_OVERLAP
|
|
74
|
+
terms with the instruction."""
|
|
75
|
+
if call.get("failure") or _ERRORISH.search(obs):
|
|
76
|
+
return True
|
|
77
|
+
if call.get("category") in ("edit", "execute", "change"):
|
|
78
|
+
return True
|
|
79
|
+
target = str(call.get("target") or "")
|
|
80
|
+
if target and ((next_message and target in next_message) or target.lower() in instruction.lower()):
|
|
81
|
+
return True
|
|
82
|
+
return len(_terms(obs) & terms) >= MIN_OVERLAP
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def transcript_of(traj, relevance: str = RELEVANCE) -> str:
|
|
86
|
+
"""The agent's steps in order: each message and its tool calls with a
|
|
87
|
+
bounded observation - in full when relevant to the question
|
|
88
|
+
(`_relevant`), a stub with a marker otherwise; `relevance="all"` keeps
|
|
89
|
+
every observation. A stub carries no observation text at all, so a
|
|
90
|
+
secret in an omitted observation cannot leak through it."""
|
|
91
|
+
return _transcript(traj, relevance)[0]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cap_observation(obs: str, limit: int = OBSERVATION, tail: int = OBSERVATION_TAIL) -> str:
|
|
95
|
+
"""An observation over the cap: its head, a marker saying how much is left out, its tail."""
|
|
96
|
+
if len(obs) <= limit:
|
|
97
|
+
return obs
|
|
98
|
+
head = limit - tail
|
|
99
|
+
return obs[:head] + f" … [{len(obs) - limit} characters left out here] … " + obs[-tail:]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _transcript(traj, relevance: str) -> tuple:
|
|
103
|
+
instruction = instruction_of(traj) if relevance != "all" else ""
|
|
104
|
+
terms = _terms(instruction)
|
|
105
|
+
steps = [s for s in traj.steps if s.source == "agent"]
|
|
106
|
+
out = []
|
|
107
|
+
kept = omitted = omitted_chars = 0
|
|
108
|
+
for i, s in enumerate(steps):
|
|
109
|
+
if s.message:
|
|
110
|
+
out.append(f"[agent] {s.message}")
|
|
111
|
+
following = next((t.message for t in steps[i + 1:] if t.message), "") or ""
|
|
112
|
+
for c in s.tool_calls or []:
|
|
113
|
+
args = c.get("arguments")
|
|
114
|
+
obs = str(c.get("observation") or "")
|
|
115
|
+
if relevance != "all" and obs and not _relevant(c, obs, instruction, terms, following):
|
|
116
|
+
omitted += 1; omitted_chars += len(obs)
|
|
117
|
+
obs = OMITTED.format(n=len(obs))
|
|
118
|
+
else:
|
|
119
|
+
if obs:
|
|
120
|
+
kept += 1
|
|
121
|
+
if len(obs) > OBSERVATION:
|
|
122
|
+
obs = cap_observation(obs)
|
|
123
|
+
out.append(f"[tool {c.get('name')}] {args if args else ''}\n[observation] {obs}".rstrip())
|
|
124
|
+
if relevance != "all" and (kept or omitted):
|
|
125
|
+
out.insert(0, KEPT_MARKER.format(kept=kept, omitted=omitted))
|
|
126
|
+
return "\n".join(out), {"kept": kept, "omitted": omitted, "omitted_chars": omitted_chars}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def verifier_text(trial_dir: str, limit: int = 8_000) -> str:
|
|
130
|
+
"""What the verifier wrote, from the files the artifact reader allows and nothing else."""
|
|
131
|
+
parts = []
|
|
132
|
+
for name in ("verifier/reward.txt", "verifier/reward-details.json", "verifier/reward.json", "verifier/report.json", "verifier/test-stdout.txt"):
|
|
133
|
+
path = os.path.join(trial_dir, name)
|
|
134
|
+
try:
|
|
135
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
136
|
+
text = fh.read(limit + 1)
|
|
137
|
+
except OSError:
|
|
138
|
+
continue
|
|
139
|
+
parts.append(f"[{name}]\n{text[:limit]}")
|
|
140
|
+
return "\n".join(parts)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def build(kind: str, traj, trial=None, trial_dir: Optional[str] = None, other=None, limit: int = LIMIT,
|
|
144
|
+
relevance: str = RELEVANCE, **extra) -> tuple:
|
|
145
|
+
"""The state of one kind, redacted, truncated: (text, truncated).
|
|
146
|
+
`relevance` is the observation policy `transcript_of` applies to the
|
|
147
|
+
built-in kinds ("question" by default, "all" keeps every observation).
|
|
148
|
+
|
|
149
|
+
`trial`: instruction + steps + final message. `failure`: the same plus
|
|
150
|
+
the recorded error and the verifier text. `final`: the final message.
|
|
151
|
+
`pair`: instruction + transcript A + transcript B (`other` is B's
|
|
152
|
+
trajectory). Any other kind is a registered builder's (`register`),
|
|
153
|
+
given the same keywords."""
|
|
154
|
+
if kind in BUILDERS:
|
|
155
|
+
text = BUILDERS[kind](traj, trial=trial, trial_dir=trial_dir, other=other, **extra)
|
|
156
|
+
elif kind == "final":
|
|
157
|
+
text = "[final message]\n" + (traj.last_agent_message or "")
|
|
158
|
+
elif kind == "pair":
|
|
159
|
+
# each transcript truncated on its own, so a long A cannot push B out of the state
|
|
160
|
+
half = max(2_000, (limit - 4_000) // 2)
|
|
161
|
+
a, ta = truncate(transcript_of(traj, relevance), half)
|
|
162
|
+
b, tb = truncate(transcript_of(other, relevance), half)
|
|
163
|
+
text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript A]\n" + a + "\n[final message A]\n" + (traj.last_agent_message or "") \
|
|
164
|
+
+ "\n\n[transcript B]\n" + b + "\n[final message B]\n" + (other.last_agent_message or "")
|
|
165
|
+
text, t = truncate(redact(text), limit)
|
|
166
|
+
return text, t or ta or tb
|
|
167
|
+
else:
|
|
168
|
+
text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript]\n" + transcript_of(traj, relevance) + "\n\n[final message]\n" + (traj.last_agent_message or "")
|
|
169
|
+
if kind == "failure":
|
|
170
|
+
err = ""
|
|
171
|
+
if trial is not None and (trial.exception_type or trial.exception_message):
|
|
172
|
+
err = f"{trial.exception_type}: {trial.exception_message}".strip(": ")
|
|
173
|
+
text += "\n\n[recorded error]\n" + (err or "none recorded")
|
|
174
|
+
text += "\n\n[verifier]\n" + (verifier_text(trial_dir) if trial_dir else "")
|
|
175
|
+
return truncate(redact(text), limit)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
# --- Purpose-built states, each a slice and never the whole transcript
|
|
179
|
+
|
|
180
|
+
CONTROL_TRANSCRIPT = 6_000 # characters of a control's own transcript kept in its state
|
|
181
|
+
|
|
182
|
+
CONTROL_PURPOSE = {
|
|
183
|
+
"nop": "nop does nothing to the task; the verifier must reject it (reward 0), and no previously-passing test may fail.",
|
|
184
|
+
"oracle": "oracle applies the task's reference solution; the verifier must accept it (reward 1).",
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def control_state(traj, trial=None, trial_dir=None, control: str = "", outcome_label: str = "", **_) -> str:
|
|
189
|
+
"""The `control` state: which control and what it is for (written here),
|
|
190
|
+
its recorded outcome and error, the verifier text, and its own transcript
|
|
191
|
+
bounded - the agent's run is never in it."""
|
|
192
|
+
err = ""
|
|
193
|
+
if trial is not None and (getattr(trial, "exception_type", "") or getattr(trial, "exception_message", "")):
|
|
194
|
+
err = f"{trial.exception_type}: {trial.exception_message}".strip(": ")
|
|
195
|
+
transcript = "none recorded"
|
|
196
|
+
if traj is not None and not getattr(traj, "error", None):
|
|
197
|
+
transcript, _cut = truncate(transcript_of(traj) or "none recorded", CONTROL_TRANSCRIPT)
|
|
198
|
+
reward = getattr(trial, "reward", None) if trial is not None else None
|
|
199
|
+
outcome = outcome_label or (getattr(trial, "outcome", "") if trial is not None else "") or "not recorded"
|
|
200
|
+
# what the reader counted, written here as sentences - the judge is never asked to count
|
|
201
|
+
counted = []
|
|
202
|
+
if trial is not None and (getattr(trial, "f2p_pass", 0) or getattr(trial, "f2p_fail", 0)):
|
|
203
|
+
counted.append(f"Tests the task expects to pass after the change: {trial.f2p_pass} passed, {trial.f2p_fail} failed.")
|
|
204
|
+
if trial is not None and (getattr(trial, "p2p_pass", 0) or getattr(trial, "p2p_fail", 0)):
|
|
205
|
+
counted.append(f"Tests that passed before the change: {trial.p2p_pass} still pass, {trial.p2p_fail} now fail.")
|
|
206
|
+
held = list(getattr(trial, "p2p_failures", None) or [])[:10]
|
|
207
|
+
if held:
|
|
208
|
+
counted.append("Previously-passing tests that failed: " + ", ".join(held) + (" …" if getattr(trial, "p2p_fail", 0) > 10 else ""))
|
|
209
|
+
return ("[control]\n" + (CONTROL_PURPOSE.get(control) or f"{control or 'a control'}: a Harbor control run") +
|
|
210
|
+
"\n\n[recorded outcome]\n" + outcome + (f" (reward {reward})" if reward is not None else "") +
|
|
211
|
+
("\n\n[test counts, from the verifier's report]\n" + "\n".join(counted) if counted else "") +
|
|
212
|
+
"\n\n[recorded error]\n" + (err or "none recorded") +
|
|
213
|
+
"\n\n[verifier]\n" + ((verifier_text(trial_dir) if trial_dir else "") or "nothing recorded") +
|
|
214
|
+
"\n\n[transcript]\n" + transcript)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def task_state(traj, instruction: str = "", **_) -> str:
|
|
218
|
+
"""The `task` state: the instruction alone - the same text for every
|
|
219
|
+
attempt at the task, and nothing of any run."""
|
|
220
|
+
return "[instruction]\n" + (instruction or (instruction_of(traj) if traj is not None else ""))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
register("control", control_state)
|
|
224
|
+
register("task", task_state)
|
nullius/judge/store.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Where a judgement lives: the workspace's own store, beside the parsed
|
|
2
|
+
jobs - ``<workspace>/judgements/<job_id>/<trial_name>.json`` for System
|
|
3
|
+
One's, ``<workspace>/judgements-llm/...`` for the LLM judge's (a trial has
|
|
4
|
+
both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the
|
|
5
|
+
run tree is Harbor's, and a re-parse of the job does not touch this.
|
|
6
|
+
|
|
7
|
+
Every version is kept. A read without a version asks for the current one
|
|
8
|
+
and falls back to the earlier ones the vocabulary still names
|
|
9
|
+
(``questions.EARLIER``), newest first, so a judgement made under v1 is
|
|
10
|
+
still shown after the bump - under its own version, which the record
|
|
11
|
+
carries - and a write always lands under the version the record names."""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
from typing import Optional
|
|
18
|
+
|
|
19
|
+
from . import client, questions as Q
|
|
20
|
+
|
|
21
|
+
_SAFE = re.compile(r"[^A-Za-z0-9._@+-]")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
ROOTS = {"system-one": "judgements", "llm": "judgements-llm"} # one directory per judgement
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _root(ws_dir: str, lane: str) -> str:
|
|
28
|
+
return os.path.join(ws_dir, ROOTS.get(lane, "judgements"))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _path(ws_dir: str, job_id: str, trial_name: str, lane: str = "system-one") -> str:
|
|
32
|
+
return os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id), _SAFE.sub("_", trial_name) + ".json")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _versions(version: Optional[str]) -> tuple:
|
|
36
|
+
return (version,) if version else (Q.VERSION, *Q.EARLIER)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _pick(data: dict, version: Optional[str]) -> Optional[dict]:
|
|
40
|
+
for v in _versions(version):
|
|
41
|
+
rec = data.get(v)
|
|
42
|
+
if rec:
|
|
43
|
+
return rec
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one") -> Optional[dict]:
|
|
48
|
+
"""The record under `version`; with none named, the current version or
|
|
49
|
+
the newest earlier one that has a record."""
|
|
50
|
+
try:
|
|
51
|
+
with open(_path(ws_dir, job_id, trial_name, lane), encoding="utf-8") as fh:
|
|
52
|
+
return _pick(json.load(fh) or {}, version)
|
|
53
|
+
except (OSError, ValueError):
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def is_current(record: Optional[dict]) -> bool:
|
|
58
|
+
"""Whether a record was asked under the current vocabulary."""
|
|
59
|
+
return bool(record) and record.get("questions") == Q.VERSION
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one") -> None:
|
|
63
|
+
"""The record under its question version; other versions kept. A
|
|
64
|
+
record's answers merge over the version's earlier answers, so judging
|
|
65
|
+
one question later does not drop the others."""
|
|
66
|
+
path = _path(ws_dir, job_id, trial_name, lane)
|
|
67
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
68
|
+
try:
|
|
69
|
+
with open(path, encoding="utf-8") as fh:
|
|
70
|
+
data = json.load(fh) or {}
|
|
71
|
+
except (OSError, ValueError):
|
|
72
|
+
data = {}
|
|
73
|
+
version = record.get("questions", Q.VERSION)
|
|
74
|
+
earlier = data.get(version) or {}
|
|
75
|
+
merged = dict(record)
|
|
76
|
+
merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})}
|
|
77
|
+
data[version] = merged
|
|
78
|
+
tmp = path + ".tmp"
|
|
79
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
80
|
+
json.dump(data, fh, indent=1)
|
|
81
|
+
os.replace(tmp, path)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one") -> dict:
|
|
85
|
+
"""Every judged trial of a job: {trial_name: record}, each the current
|
|
86
|
+
version's record or the newest earlier one, as `load` picks it."""
|
|
87
|
+
d = os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id))
|
|
88
|
+
out = {}
|
|
89
|
+
try:
|
|
90
|
+
names = os.listdir(d)
|
|
91
|
+
except OSError:
|
|
92
|
+
return out
|
|
93
|
+
for n in names:
|
|
94
|
+
if not n.endswith(".json"):
|
|
95
|
+
continue
|
|
96
|
+
try:
|
|
97
|
+
with open(os.path.join(d, n), encoding="utf-8") as fh:
|
|
98
|
+
rec = _pick(json.load(fh) or {}, version)
|
|
99
|
+
except (OSError, ValueError):
|
|
100
|
+
continue
|
|
101
|
+
if rec:
|
|
102
|
+
out[n[:-5]] = rec
|
|
103
|
+
return out
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "system-one"):
|
|
107
|
+
"""Every stored record, every version: yields (job_id, trial_name,
|
|
108
|
+
version, record). One job's when `job_id` is given."""
|
|
109
|
+
root = _root(ws_dir, lane)
|
|
110
|
+
try:
|
|
111
|
+
jobs = [_SAFE.sub("_", job_id)] if job_id else sorted(os.listdir(root))
|
|
112
|
+
except OSError:
|
|
113
|
+
return
|
|
114
|
+
for j in jobs:
|
|
115
|
+
d = os.path.join(root, j)
|
|
116
|
+
try:
|
|
117
|
+
names = sorted(os.listdir(d))
|
|
118
|
+
except OSError:
|
|
119
|
+
continue
|
|
120
|
+
for n in names:
|
|
121
|
+
if not n.endswith(".json"):
|
|
122
|
+
continue
|
|
123
|
+
try:
|
|
124
|
+
with open(os.path.join(d, n), encoding="utf-8") as fh:
|
|
125
|
+
data = json.load(fh) or {}
|
|
126
|
+
except (OSError, ValueError):
|
|
127
|
+
continue
|
|
128
|
+
for version, rec in data.items():
|
|
129
|
+
if isinstance(rec, dict):
|
|
130
|
+
yield j, n[:-5], version, rec
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def calibration(ws_dir: Optional[str]) -> Optional[dict]:
|
|
134
|
+
"""The judge's observed characters per token over this workspace's
|
|
135
|
+
stored calls (`client.calibrate`); None with no workspace or no sample."""
|
|
136
|
+
if not ws_dir:
|
|
137
|
+
return None
|
|
138
|
+
return client.calibrate(rec for _, _, _, rec in all_records(ws_dir))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def truncation_report(ws_dir: str, job_id: str, limit: Optional[int] = None) -> dict:
|
|
142
|
+
"""Which state kinds were cut to the limit over a job's stored calls,
|
|
143
|
+
and how often - counted here. `kinds[kind]` is {calls, truncated,
|
|
144
|
+
max_chars}, most-truncated first; `untyped` counts records that say
|
|
145
|
+
`truncated` but carry no per-state sizes (v1), which cannot be placed."""
|
|
146
|
+
from . import state as judge_state
|
|
147
|
+
limit = judge_state.LIMIT if limit is None else limit
|
|
148
|
+
kinds: dict = {}
|
|
149
|
+
records = truncated = untyped = 0
|
|
150
|
+
for _, _, _, rec in all_records(ws_dir, job_id):
|
|
151
|
+
records += 1
|
|
152
|
+
if rec.get("truncated"):
|
|
153
|
+
truncated += 1
|
|
154
|
+
entries = rec.get("states") or {}
|
|
155
|
+
if not entries and rec.get("truncated"):
|
|
156
|
+
untyped += 1
|
|
157
|
+
for kind, e in entries.items():
|
|
158
|
+
k = kinds.setdefault(kind, {"calls": 0, "truncated": 0, "max_chars": 0})
|
|
159
|
+
k["calls"] += 1
|
|
160
|
+
k["truncated"] += 1 if e.get("truncated") else 0
|
|
161
|
+
k["max_chars"] = max(k["max_chars"], e.get("chars") or 0)
|
|
162
|
+
return {"limit": limit, "records": records, "truncated": truncated, "untyped": untyped,
|
|
163
|
+
"kinds": dict(sorted(kinds.items(), key=lambda kv: (-kv[1]["truncated"], kv[0])))}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def pair_key(a_trial: str, b_trial: str) -> str:
|
|
167
|
+
return "pair:" + _SAFE.sub("_", a_trial) + "__vs__" + _SAFE.sub("_", b_trial)
|
nullius/migrate.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Preview and copy an existing checkout's user data into a persistent profile.
|
|
2
|
+
|
|
3
|
+
Sources are never changed. Existing catalogue entries must have identical
|
|
4
|
+
inventories and bytes; conflicting data stops migration before any writes.
|
|
5
|
+
Only explicit --apply writes. No output includes credential values.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import stat
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from nullius import paths, settings
|
|
18
|
+
|
|
19
|
+
SKIP = {".git", "__pycache__", ".pytest_cache", ".venv"}
|
|
20
|
+
PATH_SETTINGS = {"RUNS_DIR", "WORKSPACES_DIR", "HARBOR_PINS", "STANDARD_DIR", "STANDARD_DATA_DIR",
|
|
21
|
+
"SECRETS_DIR", "NULLIUS_CATALOGUE"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class CopyPlan:
|
|
26
|
+
files: list[tuple[Path, bytes, int, bytes | None]] = field(default_factory=list)
|
|
27
|
+
directories: set[Path] = field(default_factory=set)
|
|
28
|
+
catalogue_entries: dict[Path, dict | None] = field(default_factory=dict)
|
|
29
|
+
catalogue_snapshot: bool = False
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _no_symlinks(path: Path):
|
|
33
|
+
if any(p.is_symlink() for p in (path, *path.parents)):
|
|
34
|
+
raise ValueError(f"Refusing a symbolic link: {path}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _inventory(root: Path, skip=()) -> dict | None:
|
|
38
|
+
"""Relative directories (None) and file bytes, including an empty root."""
|
|
39
|
+
_no_symlinks(root)
|
|
40
|
+
if not root.exists():
|
|
41
|
+
return None
|
|
42
|
+
if not root.is_dir():
|
|
43
|
+
raise ValueError(f"Not a directory: {root}")
|
|
44
|
+
inventory = {Path("."): None}
|
|
45
|
+
|
|
46
|
+
def visit(directory):
|
|
47
|
+
for source in sorted(directory.iterdir()):
|
|
48
|
+
if source.is_symlink():
|
|
49
|
+
raise ValueError(f"Refusing a symbolic link: {source}")
|
|
50
|
+
relative = source.relative_to(root)
|
|
51
|
+
if source.is_dir():
|
|
52
|
+
if source.name not in skip:
|
|
53
|
+
inventory[relative] = None
|
|
54
|
+
visit(source)
|
|
55
|
+
elif stat.S_ISREG(source.stat().st_mode):
|
|
56
|
+
inventory[relative] = source.read_bytes()
|
|
57
|
+
else:
|
|
58
|
+
raise ValueError(f"Not a regular file: {source}")
|
|
59
|
+
|
|
60
|
+
visit(root)
|
|
61
|
+
return inventory
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _files(root: Path):
|
|
65
|
+
_no_symlinks(root)
|
|
66
|
+
if not root.exists():
|
|
67
|
+
return
|
|
68
|
+
if not root.is_dir():
|
|
69
|
+
raise ValueError(f"Not a directory: {root}")
|
|
70
|
+
|
|
71
|
+
def fail(error):
|
|
72
|
+
raise error
|
|
73
|
+
|
|
74
|
+
for directory, dirs, files in os.walk(root, followlinks=False, onerror=fail):
|
|
75
|
+
for name in dirs + sorted(files):
|
|
76
|
+
source = Path(directory, name)
|
|
77
|
+
if source.is_symlink():
|
|
78
|
+
raise ValueError(f"Refusing a symbolic link: {source}")
|
|
79
|
+
dirs[:] = sorted(d for d in dirs if d not in SKIP)
|
|
80
|
+
for name in sorted(files):
|
|
81
|
+
source = Path(directory, name)
|
|
82
|
+
if not stat.S_ISREG(source.stat().st_mode):
|
|
83
|
+
raise ValueError(f"Not a regular file: {source}")
|
|
84
|
+
yield source
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _absolute(value: str, checkout: Path) -> str:
|
|
88
|
+
p = Path(value).expanduser()
|
|
89
|
+
return os.path.abspath(p if p.is_absolute() else checkout / p)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def prepare(checkout: Path, licence_source: Path | None = None):
|
|
93
|
+
"""Return a copy plan and counts; create nothing, including during preview."""
|
|
94
|
+
checkout = checkout.expanduser().resolve()
|
|
95
|
+
if not (checkout / "app" / "nullius").is_dir():
|
|
96
|
+
raise ValueError("--from must name a Nullius checkout containing app/nullius.")
|
|
97
|
+
old_file = checkout / "app" / ".env"
|
|
98
|
+
_no_symlinks(old_file)
|
|
99
|
+
values = settings.in_file(str(old_file))
|
|
100
|
+
for name, value in list(values.items()):
|
|
101
|
+
if not settings._ENVNAME_OK.fullmatch(name) or any(c in value for c in "\0\r\n"):
|
|
102
|
+
raise ValueError("The old settings file contains an invalid variable name or value.")
|
|
103
|
+
if value and name in PATH_SETTINGS:
|
|
104
|
+
values[name] = _absolute(value, checkout)
|
|
105
|
+
elif value and name in ("JOB_ROOTS", "DATASET_ROOTS"):
|
|
106
|
+
values[name] = os.pathsep.join(_absolute(v, checkout) for v in value.split(os.pathsep) if v)
|
|
107
|
+
# Explicit workspace locations stay in use, without rewriting their records.
|
|
108
|
+
old_workspaces = None
|
|
109
|
+
if not values.get("WORKSPACES_DIR"):
|
|
110
|
+
values.pop("WORKSPACES_DIR", None)
|
|
111
|
+
old_workspaces = (checkout / "workspaces").resolve()
|
|
112
|
+
# Large raw runs stay where they are. Their location is made explicit.
|
|
113
|
+
if not values.get("RUNS_DIR") and (checkout / "runs").is_dir():
|
|
114
|
+
values["RUNS_DIR"] = str(checkout / "runs")
|
|
115
|
+
if licence_source is not None:
|
|
116
|
+
values.pop("SECRETS_DIR", None)
|
|
117
|
+
|
|
118
|
+
# Resolve selected storage roots once; links inside their contents remain refused.
|
|
119
|
+
target = Path(paths.config_dir()).expanduser().resolve() / "settings.json"
|
|
120
|
+
catalogue_target = Path(paths.catalogue_dir()).expanduser().resolve()
|
|
121
|
+
workspaces_target = Path(paths.workspaces_dir()).expanduser().resolve()
|
|
122
|
+
secrets_target = Path(paths.secrets_dir()).expanduser().resolve()
|
|
123
|
+
_no_symlinks(target)
|
|
124
|
+
held = settings.in_file(str(target))
|
|
125
|
+
for name, value in values.items():
|
|
126
|
+
if name in held and held[name] != value:
|
|
127
|
+
raise ValueError(f"Existing setting differs: {name}. Choose a separate NULLIUS_HOME or resolve it first.")
|
|
128
|
+
merged = {**held, **values}
|
|
129
|
+
pending = CopyPlan()
|
|
130
|
+
counts = {"settings": len(values), "catalogue_entries": 0, "workspace_files": 0, "licence_files": 0}
|
|
131
|
+
|
|
132
|
+
def destination(dest):
|
|
133
|
+
_no_symlinks(dest)
|
|
134
|
+
if dest.resolve().is_relative_to(checkout):
|
|
135
|
+
raise ValueError("Migration destinations must be outside the source checkout.")
|
|
136
|
+
|
|
137
|
+
def add(dest, data, mode=0o600):
|
|
138
|
+
dest = Path(dest)
|
|
139
|
+
destination(dest)
|
|
140
|
+
if dest.exists():
|
|
141
|
+
if not dest.is_file() or dest.read_bytes() != data:
|
|
142
|
+
raise ValueError(f"Destination already contains different data: {dest}")
|
|
143
|
+
else:
|
|
144
|
+
pending.files.append((dest, data, mode, None))
|
|
145
|
+
|
|
146
|
+
if merged != held or (merged and not target.exists()):
|
|
147
|
+
# Merging non-conflicting settings is the one allowed replacement.
|
|
148
|
+
data = (json.dumps(merged, indent=2, ensure_ascii=False) + "\n").encode()
|
|
149
|
+
destination(target)
|
|
150
|
+
pending.files.append((target, data, 0o600, target.read_bytes() if target.exists() else None))
|
|
151
|
+
|
|
152
|
+
bundled = Path(__file__).resolve().parent / "catalogue"
|
|
153
|
+
source_catalogue = checkout / "app" / "nullius" / "catalogue"
|
|
154
|
+
pending.catalogue_snapshot = source_catalogue.resolve() == bundled.resolve()
|
|
155
|
+
for kind in ("benchmarks", "agents", "features", "capabilities"):
|
|
156
|
+
parent = source_catalogue / kind
|
|
157
|
+
_no_symlinks(parent)
|
|
158
|
+
if not parent.exists():
|
|
159
|
+
continue
|
|
160
|
+
for entry in sorted(parent.iterdir()):
|
|
161
|
+
_no_symlinks(entry)
|
|
162
|
+
if entry.name in SKIP or not entry.is_dir():
|
|
163
|
+
continue
|
|
164
|
+
inventory = _inventory(entry, skip=SKIP)
|
|
165
|
+
if any(p.name == ".env" for p in inventory):
|
|
166
|
+
raise ValueError("A catalogue entry contains .env; move its credentials to Settings before migration.")
|
|
167
|
+
expected = bundled / kind / entry.name
|
|
168
|
+
if not pending.catalogue_snapshot and inventory == _inventory(expected, skip=SKIP):
|
|
169
|
+
continue
|
|
170
|
+
counts["catalogue_entries"] += 1
|
|
171
|
+
dest = catalogue_target / kind / entry.name
|
|
172
|
+
destination(dest)
|
|
173
|
+
existing = _inventory(dest)
|
|
174
|
+
if existing is not None and inventory != existing:
|
|
175
|
+
raise ValueError(f"Destination catalogue entry differs: {dest}")
|
|
176
|
+
pending.catalogue_entries[dest] = existing
|
|
177
|
+
if existing is not None:
|
|
178
|
+
continue
|
|
179
|
+
for relative, data in inventory.items():
|
|
180
|
+
if data is None:
|
|
181
|
+
pending.directories.add(dest / relative)
|
|
182
|
+
else:
|
|
183
|
+
source = entry / relative
|
|
184
|
+
add(dest / relative, data, 0o700 if source.stat().st_mode & 0o111 else 0o600)
|
|
185
|
+
|
|
186
|
+
for source in _files(old_workspaces) if old_workspaces is not None else ():
|
|
187
|
+
data = source.read_bytes()
|
|
188
|
+
if source.name == "workspace.json":
|
|
189
|
+
try:
|
|
190
|
+
record = json.loads(data)
|
|
191
|
+
record["roots"] = [_absolute(v, checkout) for v in record["roots"]]
|
|
192
|
+
data = (json.dumps(record, indent=1, ensure_ascii=False) + "\n").encode()
|
|
193
|
+
except (ValueError, KeyError, TypeError):
|
|
194
|
+
raise ValueError(f"Invalid workspace record: {source}") from None
|
|
195
|
+
add(workspaces_target / source.relative_to(old_workspaces), data)
|
|
196
|
+
counts["workspace_files"] += 1
|
|
197
|
+
if licence_source is not None:
|
|
198
|
+
licence_source = licence_source.expanduser().resolve()
|
|
199
|
+
if not licence_source.is_dir():
|
|
200
|
+
raise ValueError("--secrets must name an existing licence directory.")
|
|
201
|
+
for source in _files(licence_source):
|
|
202
|
+
add(secrets_target / source.relative_to(licence_source), source.read_bytes())
|
|
203
|
+
counts["licence_files"] += 1
|
|
204
|
+
return pending, counts
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def apply(pending):
|
|
208
|
+
"""Write the prepared copy using private files. Original data is untouched."""
|
|
209
|
+
import tempfile
|
|
210
|
+
for dest, inventory in pending.catalogue_entries.items():
|
|
211
|
+
if _inventory(dest) != inventory:
|
|
212
|
+
raise ValueError(f"Destination changed since preview: {dest}")
|
|
213
|
+
for dest, _, _, previous in pending.files:
|
|
214
|
+
_no_symlinks(dest)
|
|
215
|
+
if (dest.read_bytes() if dest.exists() else None) != previous:
|
|
216
|
+
raise ValueError(f"Destination changed since preview: {dest}")
|
|
217
|
+
|
|
218
|
+
def mkdir(parent):
|
|
219
|
+
missing = []
|
|
220
|
+
while not parent.exists():
|
|
221
|
+
missing.append(parent)
|
|
222
|
+
parent = parent.parent
|
|
223
|
+
for directory in reversed(missing):
|
|
224
|
+
directory.mkdir(mode=0o700, exist_ok=True)
|
|
225
|
+
|
|
226
|
+
for directory in sorted(pending.directories):
|
|
227
|
+
mkdir(directory)
|
|
228
|
+
for dest, data, mode, _ in pending.files:
|
|
229
|
+
mkdir(dest.parent)
|
|
230
|
+
fd, temporary = tempfile.mkstemp(prefix=".migrate-", dir=dest.parent)
|
|
231
|
+
try:
|
|
232
|
+
os.fchmod(fd, mode)
|
|
233
|
+
with os.fdopen(fd, "wb") as stream:
|
|
234
|
+
stream.write(data)
|
|
235
|
+
os.replace(temporary, dest)
|
|
236
|
+
finally:
|
|
237
|
+
if os.path.exists(temporary):
|
|
238
|
+
os.unlink(temporary)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def main(argv=None) -> int:
|
|
242
|
+
parser = argparse.ArgumentParser(description="Copy legacy settings, custom catalogue entries and workspaces; preview by default.")
|
|
243
|
+
parser.add_argument("--from", dest="checkout", type=Path, required=True, help="existing Nullius checkout (left unchanged)")
|
|
244
|
+
parser.add_argument("--secrets", type=Path, help="also copy a legacy benchmark licence directory")
|
|
245
|
+
parser.add_argument("--apply", action="store_true", help="write the previewed copy; refuse conflicting destination data")
|
|
246
|
+
args = parser.parse_args(argv)
|
|
247
|
+
try:
|
|
248
|
+
pending, counts = prepare(args.checkout, args.secrets)
|
|
249
|
+
if args.apply:
|
|
250
|
+
apply(pending)
|
|
251
|
+
except (OSError, ValueError) as error:
|
|
252
|
+
print(f"Migration stopped: {error}", file=sys.stderr)
|
|
253
|
+
return 1
|
|
254
|
+
print("Copied." if args.apply else "Preview only. Add --apply to copy; source files are kept.")
|
|
255
|
+
if pending.catalogue_snapshot:
|
|
256
|
+
print("Source is the running catalogue; all entries are copied as persistent overlays "
|
|
257
|
+
"that retain these versions across upgrades.")
|
|
258
|
+
for name, count in counts.items():
|
|
259
|
+
print(f"{name}: {count}")
|
|
260
|
+
print(f"directories_to_create: {len(pending.directories)}")
|
|
261
|
+
print(f"files_to_write: {len(pending.files)}")
|
|
262
|
+
print(f"bytes_to_write: {sum(len(data) for _, data, _, _ in pending.files)}")
|
|
263
|
+
print(f"settings: {paths.settings_file()}")
|
|
264
|
+
return 0
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
if __name__ == "__main__":
|
|
268
|
+
raise SystemExit(main())
|