nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
"""A standard runner snapshot made from this runtime and its effective catalogue.
|
|
2
|
+
|
|
3
|
+
Only backend Python modules, definition documents and their local resources
|
|
4
|
+
travel. The installation's neighbours, user configuration, licences and SSH
|
|
5
|
+
keys are never inputs. Source checkouts and installed wheels use the same
|
|
6
|
+
content address, including executable bits; neither Git nor the viewer is
|
|
7
|
+
needed on the target. Credential transport belongs to ``execution``.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ast
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import shutil
|
|
17
|
+
import stat
|
|
18
|
+
import tempfile
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from nullius import paths
|
|
22
|
+
from nullius.standard import definitions
|
|
23
|
+
|
|
24
|
+
PACKAGE = Path(__file__).resolve().parents[1]
|
|
25
|
+
FORMAT = 1
|
|
26
|
+
_KINDS = {"benchmarks": ("benchmark.json", "standard.json"),
|
|
27
|
+
"agents": ("agent.json",), "capabilities": ("capability.json",)}
|
|
28
|
+
_DOCUMENTS = {"feature.json", "benchmark.json", "standard.json", "agent.json", "capability.json",
|
|
29
|
+
"upstream-settings.json", "images.json", "data-source.json"}
|
|
30
|
+
_PRIVATE = {"credentials", "credentials.json", "secret", "secret.json", "secrets", "secrets.json", "settings.json",
|
|
31
|
+
"id_rsa", "id_ed25519", "id_ecdsa", "id_dsa", "node_modules", "__pycache__"}
|
|
32
|
+
_KEY = re.compile(r"^[a-z][a-z0-9-]{0,39}$")
|
|
33
|
+
_HERE = re.compile(r"\{here\}/([A-Za-z0-9_./-]+)")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _safe(relative: str) -> bool:
|
|
37
|
+
p = Path(relative)
|
|
38
|
+
return (bool(relative) and not p.is_absolute() and "\\" not in relative
|
|
39
|
+
and all(not s.startswith(".") and s.lower() not in _PRIVATE for s in p.parts)
|
|
40
|
+
and p.suffix.lower() not in {".pem", ".key", ".p12", ".pfx", ".pyc"})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read(root: Path, relative: str) -> tuple[bytes, int]:
|
|
44
|
+
"""Read a regular file below root, refusing symlinks at every segment."""
|
|
45
|
+
if not _safe(relative):
|
|
46
|
+
raise ValueError(f"not a runner resource: {relative}")
|
|
47
|
+
p = root
|
|
48
|
+
for segment in Path(relative).parts:
|
|
49
|
+
p = p / segment
|
|
50
|
+
if p.is_symlink():
|
|
51
|
+
raise ValueError(f"runner resources cannot be symlinks: {relative}")
|
|
52
|
+
mode = p.stat().st_mode
|
|
53
|
+
if not stat.S_ISREG(mode):
|
|
54
|
+
raise ValueError(f"runner resource is not a file: {relative}")
|
|
55
|
+
return p.read_bytes(), 0o755 if mode & 0o111 else 0o644
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _python(root: Path):
|
|
59
|
+
"""Backend/glue code only: never follow a directory link or copy caches."""
|
|
60
|
+
for directory, folders, files in os.walk(root, followlinks=False):
|
|
61
|
+
folders[:] = sorted(n for n in folders if _safe(n) and n != "_resources"
|
|
62
|
+
and not (Path(directory) / n).is_symlink())
|
|
63
|
+
for name in sorted(files):
|
|
64
|
+
if name.endswith(".py") and _safe(name):
|
|
65
|
+
yield (Path(directory) / name).relative_to(root).as_posix()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _resources(d: dict) -> set[str]:
|
|
69
|
+
"""Local files named by commands, model templates and image contexts.
|
|
70
|
+
|
|
71
|
+
Walk version blocks too. Licence declarations name runtime destinations,
|
|
72
|
+
not resources; they are deliberately not interpreted as local paths.
|
|
73
|
+
"""
|
|
74
|
+
out: set[str] = set()
|
|
75
|
+
def visit(value):
|
|
76
|
+
if isinstance(value, str):
|
|
77
|
+
out.update(_HERE.findall(value))
|
|
78
|
+
elif isinstance(value, list):
|
|
79
|
+
for v in value:
|
|
80
|
+
visit(v)
|
|
81
|
+
elif isinstance(value, dict):
|
|
82
|
+
for k, v in value.items():
|
|
83
|
+
if k != "needs_secrets":
|
|
84
|
+
visit(v)
|
|
85
|
+
model = value.get("model")
|
|
86
|
+
if isinstance(model, dict) and isinstance(model.get("files"), dict):
|
|
87
|
+
out.update(v for v in model["files"].values() if isinstance(v, str))
|
|
88
|
+
environments = value.get("environments")
|
|
89
|
+
if isinstance(environments, dict):
|
|
90
|
+
for environment in environments.values():
|
|
91
|
+
if isinstance(environment, dict):
|
|
92
|
+
if isinstance(environment.get("dockerfile"), str):
|
|
93
|
+
out.add(environment["dockerfile"])
|
|
94
|
+
out.update(v for v in environment.get("files", []) if isinstance(v, str))
|
|
95
|
+
visit(d)
|
|
96
|
+
return out
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _dataset_resources(d: dict) -> set[str]:
|
|
100
|
+
"""Dataset files installed beside each effective benchmark version.
|
|
101
|
+
|
|
102
|
+
Upstream checkout paths and downloaded datasets belong to the installer,
|
|
103
|
+
not the catalogue snapshot. Resolve versions to retain inherited upstreams
|
|
104
|
+
and datasets exactly as the runner sees them.
|
|
105
|
+
"""
|
|
106
|
+
out = set()
|
|
107
|
+
for version in definitions.versions(d):
|
|
108
|
+
bench = definitions.resolve(d, version)
|
|
109
|
+
if bench is None or (bench.get("upstream") or {}).get("repository"):
|
|
110
|
+
continue
|
|
111
|
+
for dataset in bench.get("datasets") or []:
|
|
112
|
+
if isinstance(dataset, dict) and not dataset.get("url"):
|
|
113
|
+
path = dataset.get("path")
|
|
114
|
+
if isinstance(path, str) and path:
|
|
115
|
+
out.add(path)
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _imports(root: Path, relative: str, content: bytes) -> set[str]:
|
|
120
|
+
"""Follow local Python imports from a declared driver without sweeping its folder."""
|
|
121
|
+
if not relative.endswith(".py"):
|
|
122
|
+
return set()
|
|
123
|
+
try:
|
|
124
|
+
tree = ast.parse(content)
|
|
125
|
+
except (SyntaxError, ValueError):
|
|
126
|
+
return set() # Python itself reports invalid source when run
|
|
127
|
+
found = set()
|
|
128
|
+
for node in ast.walk(tree):
|
|
129
|
+
names = []
|
|
130
|
+
base = Path(relative).parent
|
|
131
|
+
if isinstance(node, ast.Import):
|
|
132
|
+
names = [n.name for n in node.names]
|
|
133
|
+
elif isinstance(node, ast.ImportFrom):
|
|
134
|
+
if node.level:
|
|
135
|
+
for _ in range(node.level - 1):
|
|
136
|
+
base = base.parent
|
|
137
|
+
prefix = (node.module + ".") if node.module else ""
|
|
138
|
+
names = ([node.module] if node.module else []) + [prefix + n.name for n in node.names if n.name != "*"]
|
|
139
|
+
for name in names:
|
|
140
|
+
for origin in (base, Path()):
|
|
141
|
+
module = origin / name.replace(".", "/")
|
|
142
|
+
for candidate in (module.with_suffix(".py"), module / "__init__.py"):
|
|
143
|
+
r = candidate.as_posix()
|
|
144
|
+
if _safe(r) and (root / r).is_file():
|
|
145
|
+
found.add(r)
|
|
146
|
+
for parent in module.parents:
|
|
147
|
+
r = (parent / "__init__.py").as_posix()
|
|
148
|
+
if _safe(r) and (root / r).is_file():
|
|
149
|
+
found.add(r)
|
|
150
|
+
return found
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _entry(root: Path, *, kit: bool = False) -> dict[str, tuple[bytes, int]]:
|
|
154
|
+
if root.is_symlink():
|
|
155
|
+
raise ValueError("catalogue entries cannot be symlinks")
|
|
156
|
+
files = {}
|
|
157
|
+
resources = set(_python(root)) if kit else set()
|
|
158
|
+
for name in sorted(_DOCUMENTS):
|
|
159
|
+
if (root / name).exists():
|
|
160
|
+
content = _read(root, name)
|
|
161
|
+
d = json.loads(content[0])
|
|
162
|
+
if name in {"standard.json", "agent.json", "capability.json", "feature.json", "benchmark.json"} and not isinstance(d, dict):
|
|
163
|
+
raise ValueError(f"{name}: expected a definition object")
|
|
164
|
+
files[name] = content
|
|
165
|
+
if name in {"standard.json", "agent.json", "capability.json", "feature.json"}:
|
|
166
|
+
resources.update(_resources(d))
|
|
167
|
+
if name == "standard.json":
|
|
168
|
+
resources.update(_dataset_resources(d))
|
|
169
|
+
while resources - files.keys():
|
|
170
|
+
name = min(resources - files.keys())
|
|
171
|
+
files[name] = _read(root, name)
|
|
172
|
+
resources.update(_imports(root, name, files[name][0]))
|
|
173
|
+
return files
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _cached(destination: Path, marker: str, files: dict) -> bool:
|
|
177
|
+
"""Do not reuse or transfer a cache into which another file was written."""
|
|
178
|
+
if not destination.exists():
|
|
179
|
+
return False
|
|
180
|
+
manifest = destination / "manifest.json"
|
|
181
|
+
expected = {"manifest.json", *("app/nullius/" + name for name in files)}
|
|
182
|
+
found = set()
|
|
183
|
+
if destination.is_symlink():
|
|
184
|
+
raise ValueError("runner snapshot cache cannot be a symlink")
|
|
185
|
+
for directory, folders, names in os.walk(destination, followlinks=False):
|
|
186
|
+
for name in folders + names:
|
|
187
|
+
p = Path(directory) / name
|
|
188
|
+
if p.is_symlink():
|
|
189
|
+
raise ValueError("runner snapshot cache contains a symlink; remove " + str(destination))
|
|
190
|
+
found.update((Path(directory) / name).relative_to(destination).as_posix() for name in names)
|
|
191
|
+
valid = found == expected and manifest.read_text() == marker
|
|
192
|
+
if valid:
|
|
193
|
+
for name, value in files.items():
|
|
194
|
+
if _read(destination / "app/nullius", name) != value:
|
|
195
|
+
valid = False
|
|
196
|
+
break
|
|
197
|
+
if not valid:
|
|
198
|
+
raise ValueError("runner snapshot cache has changed; remove " + str(destination))
|
|
199
|
+
return True
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def build(cache: str, *, package: Path | None = None, catalogue: str | None = None) -> tuple[str, str]:
|
|
203
|
+
"""Return (content ID, snapshot root), with ``app/nullius`` below it.
|
|
204
|
+
|
|
205
|
+
A user entry replaces the whole bundled entry, as the normal catalogue
|
|
206
|
+
loader does. Nothing is created in either input tree. The staged tree is
|
|
207
|
+
replaced atomically, and the address covers the exact bytes being sent.
|
|
208
|
+
"""
|
|
209
|
+
package = Path(package) if package is not None else PACKAGE
|
|
210
|
+
if package.is_symlink():
|
|
211
|
+
raise ValueError("the runtime package cannot be a symlink")
|
|
212
|
+
files = {}
|
|
213
|
+
for relative in _python(package):
|
|
214
|
+
pieces = Path(relative).parts
|
|
215
|
+
if ((len(pieces) > 2 and pieces[0] == "catalogue" and pieces[1] in _KINDS)
|
|
216
|
+
or pieces[:2] == ("standard", "kits")):
|
|
217
|
+
continue
|
|
218
|
+
files[relative] = _read(package, relative)
|
|
219
|
+
if "standard/cli.py" not in files or "__init__.py" not in files:
|
|
220
|
+
raise ValueError("the standard runtime package is incomplete")
|
|
221
|
+
bundled = package / "catalogue"
|
|
222
|
+
user = Path(catalogue if catalogue is not None else paths.catalogue_dir())
|
|
223
|
+
if user.is_symlink() or bundled.is_symlink():
|
|
224
|
+
raise ValueError("catalogue roots cannot be symlinks")
|
|
225
|
+
for kind in _KINDS:
|
|
226
|
+
entries = {}
|
|
227
|
+
for catalogue_root in (bundled, user):
|
|
228
|
+
for directory in (("features", "capabilities") if kind == "capabilities" else (kind,)):
|
|
229
|
+
base = catalogue_root / directory
|
|
230
|
+
if base.is_symlink():
|
|
231
|
+
raise ValueError("catalogue folders cannot be symlinks")
|
|
232
|
+
if not base.is_dir():
|
|
233
|
+
continue
|
|
234
|
+
for entry in sorted(base.iterdir()):
|
|
235
|
+
if (_KEY.fullmatch(entry.name) and entry.is_dir()
|
|
236
|
+
and sorted(p.name for p in entry.iterdir()) != ["__pycache__"]):
|
|
237
|
+
entries[entry.name] = entry
|
|
238
|
+
for name, entry in sorted(entries.items()):
|
|
239
|
+
try:
|
|
240
|
+
contents = _entry(entry)
|
|
241
|
+
except json.JSONDecodeError:
|
|
242
|
+
continue # Unfinished JSON drafts are not runnable catalogue entries.
|
|
243
|
+
files.update({f"catalogue/{kind}/{name}/{r}": value for r, value in contents.items()})
|
|
244
|
+
kits = package / "standard" / "kits"
|
|
245
|
+
if kits.is_dir():
|
|
246
|
+
if kits.is_symlink():
|
|
247
|
+
raise ValueError("the kit folder cannot be a symlink")
|
|
248
|
+
for entry in sorted(kits.iterdir()):
|
|
249
|
+
if _KEY.fullmatch(entry.name) and (entry / "standard.json").is_file():
|
|
250
|
+
files.update({f"standard/kits/{entry.name}/{r}": value for r, value in _entry(entry, kit=True).items()})
|
|
251
|
+
digest = hashlib.sha256(f"nullius-standard-payload-v{FORMAT}\0".encode())
|
|
252
|
+
manifest = {}
|
|
253
|
+
for name, (content, mode) in sorted(files.items()):
|
|
254
|
+
manifest[name] = {"sha256": hashlib.sha256(content).hexdigest(), "mode": mode}
|
|
255
|
+
identity = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode()
|
|
256
|
+
digest.update(identity)
|
|
257
|
+
key = "v%d-%s" % (FORMAT, digest.hexdigest())
|
|
258
|
+
destination = Path(cache) / key
|
|
259
|
+
marker = json.dumps({"format": FORMAT, "files": manifest}, sort_keys=True, indent=2) + "\n"
|
|
260
|
+
if _cached(destination, marker, files):
|
|
261
|
+
return key, str(destination)
|
|
262
|
+
Path(cache).mkdir(parents=True, exist_ok=True)
|
|
263
|
+
staging = Path(tempfile.mkdtemp(prefix=".payload-", dir=cache))
|
|
264
|
+
try:
|
|
265
|
+
for name, (content, mode) in sorted(files.items()):
|
|
266
|
+
target = staging / "app" / "nullius" / name
|
|
267
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
268
|
+
target.write_bytes(content)
|
|
269
|
+
target.chmod(mode)
|
|
270
|
+
(staging / "manifest.json").write_text(marker)
|
|
271
|
+
try:
|
|
272
|
+
staging.rename(destination)
|
|
273
|
+
except OSError:
|
|
274
|
+
if not _cached(destination, marker, files):
|
|
275
|
+
raise
|
|
276
|
+
finally:
|
|
277
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
278
|
+
return key, str(destination)
|
nullius/standard/pins.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Compare a benchmark's image pins with registry tags, without pulling or
|
|
2
|
+
changing anything. Read images.json, or string literals in driver.py without
|
|
3
|
+
executing it. Registry V2 HEAD requests read manifest digests; anonymous
|
|
4
|
+
Bearer challenges are supported, private credentials are never loaded.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from http.client import HTTPException
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from urllib.error import HTTPError, URLError
|
|
14
|
+
from urllib.parse import urlencode, urlsplit
|
|
15
|
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
16
|
+
|
|
17
|
+
TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
|
|
18
|
+
DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
|
|
19
|
+
REPOSITORY = re.compile(r"[a-z0-9]+(?:[._-]+[a-z0-9]+)*(?:/[a-z0-9]+(?:[._-]+[a-z0-9]+)*)*")
|
|
20
|
+
ACCEPT = ", ".join(("application/vnd.oci.image.index.v1+json", "application/vnd.oci.image.manifest.v1+json",
|
|
21
|
+
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
22
|
+
"application/vnd.docker.distribution.manifest.v2+json"))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def references(folder: str) -> list:
|
|
26
|
+
"""(name, reference) in file order. images.json takes precedence."""
|
|
27
|
+
root = Path(folder)
|
|
28
|
+
if (root / "images.json").exists():
|
|
29
|
+
data = json.loads((root / "images.json").read_text())
|
|
30
|
+
if not isinstance(data, dict) or not all(isinstance(v, str) for v in data.values()):
|
|
31
|
+
raise ValueError("images.json must map names to pinned image references")
|
|
32
|
+
return list(data.items())
|
|
33
|
+
tree = ast.parse((root / "driver.py").read_text())
|
|
34
|
+
strings = sorted((n for n in ast.walk(tree) if isinstance(n, ast.Constant) and isinstance(n.value, str)
|
|
35
|
+
and "@sha256:" in n.value), key=lambda n: (n.lineno, n.col_offset))
|
|
36
|
+
return [(s, s) for s in dict.fromkeys(n.value for n in strings)]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def target(reference: str, tag: str | None = None) -> tuple:
|
|
40
|
+
"""Registry, repository, tag and pinned digest. Docker Hub is implicit;
|
|
41
|
+
--tag overrides an embedded tag, otherwise an untagged pin uses latest."""
|
|
42
|
+
name, _, digest = reference.rpartition("@")
|
|
43
|
+
if not DIGEST.fullmatch(digest):
|
|
44
|
+
raise ValueError("expected an image pinned by sha256 digest")
|
|
45
|
+
embedded = None
|
|
46
|
+
if ":" in name.rsplit("/", 1)[-1]:
|
|
47
|
+
name, embedded = name.rsplit(":", 1)
|
|
48
|
+
tag = tag if tag is not None else embedded or "latest"
|
|
49
|
+
if not TAG.fullmatch(tag):
|
|
50
|
+
raise ValueError("invalid image tag")
|
|
51
|
+
first, slash, rest = name.partition("/")
|
|
52
|
+
if slash and ("." in first or ":" in first or first == "localhost"):
|
|
53
|
+
registry, repository = first, rest
|
|
54
|
+
else:
|
|
55
|
+
registry, repository = "docker.io", name
|
|
56
|
+
if not re.fullmatch(r"[A-Za-z0-9.-]+(?::[0-9]+)?", registry) or not REPOSITORY.fullmatch(repository):
|
|
57
|
+
raise ValueError("invalid registry or repository name")
|
|
58
|
+
if registry in ("docker.io", "index.docker.io", "registry-1.docker.io"):
|
|
59
|
+
registry = "registry-1.docker.io"
|
|
60
|
+
if "/" not in repository:
|
|
61
|
+
repository = "library/" + repository
|
|
62
|
+
return registry, repository, tag, digest
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class _NoRedirect(HTTPRedirectHandler):
|
|
66
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
67
|
+
# A registry's bearer token must not follow a redirect to another service.
|
|
68
|
+
raise ValueError("registry redirect not followed")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _token(challenge: str, repository: str, open_) -> str:
|
|
72
|
+
kind, _, fields = challenge.partition(" ")
|
|
73
|
+
params = dict(re.findall(r'(\w+)="([^"\r\n]*)"', fields))
|
|
74
|
+
realm = params.get("realm", "")
|
|
75
|
+
url = urlsplit(realm)
|
|
76
|
+
if kind.lower() != "bearer" or url.scheme != "https" or not url.hostname or url.username or url.password or url.fragment:
|
|
77
|
+
raise ValueError("registry requires unsupported authentication")
|
|
78
|
+
query = urlencode({"service": params.get("service", ""), "scope": f"repository:{repository}:pull"})
|
|
79
|
+
try:
|
|
80
|
+
with open_(Request(realm + ("&" if url.query else "?") + query), timeout=15) as response:
|
|
81
|
+
data = json.loads(response.read(65537))
|
|
82
|
+
except HTTPError as e:
|
|
83
|
+
e.close()
|
|
84
|
+
raise ValueError(f"token service returned HTTP {e.code}; anonymous access unavailable") from None
|
|
85
|
+
except (ValueError, UnicodeError):
|
|
86
|
+
raise ValueError("invalid token service response") from None
|
|
87
|
+
token = (data.get("token") or data.get("access_token")) if isinstance(data, dict) else None
|
|
88
|
+
if not isinstance(token, str) or not token or any(not 33 <= ord(c) <= 126 for c in token):
|
|
89
|
+
raise ValueError("token service returned no usable bearer token")
|
|
90
|
+
return token
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def current(registry: str, repository: str, tag: str, open_=None) -> str:
|
|
94
|
+
"""Ask only for manifest headers. A missing tag raises HTTPError(404);
|
|
95
|
+
auth-service failures must not be mistaken for a missing manifest."""
|
|
96
|
+
open_ = open_ or build_opener(_NoRedirect()).open
|
|
97
|
+
url = f"https://{registry}/v2/{repository}/manifests/{tag}"
|
|
98
|
+
headers = {"Accept": ACCEPT}
|
|
99
|
+
try:
|
|
100
|
+
response = open_(Request(url, headers=headers, method="HEAD"), timeout=15)
|
|
101
|
+
except HTTPError as e:
|
|
102
|
+
if e.code != 401:
|
|
103
|
+
raise
|
|
104
|
+
challenge = e.headers.get("WWW-Authenticate", "")
|
|
105
|
+
e.close()
|
|
106
|
+
headers["Authorization"] = "Bearer " + _token(challenge, repository, open_)
|
|
107
|
+
response = open_(Request(url, headers=headers, method="HEAD"), timeout=15)
|
|
108
|
+
with response:
|
|
109
|
+
digest = response.headers.get("Docker-Content-Digest", "")
|
|
110
|
+
if not DIGEST.fullmatch(digest):
|
|
111
|
+
raise ValueError("registry returned no valid sha256 manifest digest")
|
|
112
|
+
return digest
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def check(reference: str, tag: str | None = None) -> tuple:
|
|
116
|
+
"""(state, detail): same, moved, gone, or not checked. Never include
|
|
117
|
+
response bodies, authentication URLs or credentials in diagnostics."""
|
|
118
|
+
try:
|
|
119
|
+
registry, repository, tag, pinned = target(reference, tag)
|
|
120
|
+
found = current(registry, repository, tag)
|
|
121
|
+
return ("same", f"{tag}: {pinned}") if found == pinned else ("moved", f"{tag}: {pinned} -> {found}")
|
|
122
|
+
except HTTPError as e:
|
|
123
|
+
e.close()
|
|
124
|
+
if e.code == 404:
|
|
125
|
+
return "gone", f"{tag}: registry returned HTTP 404"
|
|
126
|
+
return "not checked", f"registry returned HTTP {e.code}"
|
|
127
|
+
except (URLError, OSError, HTTPException):
|
|
128
|
+
return "not checked", "registry connection failed or timed out"
|
|
129
|
+
except ValueError as e:
|
|
130
|
+
return "not checked", str(e)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The ports of version 1: what each one promises, and the kit that proves it.
|
|
2
|
+
|
|
3
|
+
A port's name carries its version. Its contract never changes under the same
|
|
4
|
+
name: a different contract is a new version, and a definition naming the old
|
|
5
|
+
one keeps meaning the old one. ``label`` is what a page calls it, in words a
|
|
6
|
+
person needs no glossary for; the port's name stays beside it. ``kit`` is the folder under ``standard/kits/``
|
|
7
|
+
holding the tiny benchmark that proves the port, or None while a port is
|
|
8
|
+
defined ahead of its first benchmark.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from . import names
|
|
15
|
+
|
|
16
|
+
NAME = re.compile(r"^(agent|cap)\.[a-z][a-z0-9-]{0,31}/v[1-9][0-9]{0,2}$")
|
|
17
|
+
|
|
18
|
+
PORTS = {
|
|
19
|
+
"agent.task/v1": {
|
|
20
|
+
"label": "One task",
|
|
21
|
+
"family": "agent",
|
|
22
|
+
"summary": "One prompt in, one turn; the answer is the agent's final message.",
|
|
23
|
+
"kit": "agent-task-v1",
|
|
24
|
+
},
|
|
25
|
+
"agent.workspace/v1": {
|
|
26
|
+
"label": "Workspace files",
|
|
27
|
+
"family": "agent",
|
|
28
|
+
"summary": "The session's working directory is the unit's workspace; the agent reads and writes files there.",
|
|
29
|
+
"kit": "agent-workspace-v1",
|
|
30
|
+
},
|
|
31
|
+
"agent.tools/v1": {
|
|
32
|
+
"label": "MCP tools",
|
|
33
|
+
"family": "agent",
|
|
34
|
+
"summary": "The agent calls the tools of the MCP servers given to its session.",
|
|
35
|
+
"kit": "agent-tools-v1",
|
|
36
|
+
},
|
|
37
|
+
"agent.conversation/v1": {
|
|
38
|
+
"label": "Conversation",
|
|
39
|
+
"family": "agent",
|
|
40
|
+
"summary": "Several prompts in one session, each answered in its own turn, the earlier ones remembered.",
|
|
41
|
+
"kit": "agent-conversation-v1",
|
|
42
|
+
},
|
|
43
|
+
"agent.search/v1": {
|
|
44
|
+
"label": "Search",
|
|
45
|
+
"family": "agent",
|
|
46
|
+
"summary": "Search an HTTP collection, retrieve a source and answer with its evidence using the agent's available tools.",
|
|
47
|
+
"kit": "agent-search-v1",
|
|
48
|
+
},
|
|
49
|
+
"agent.browser/v1": {
|
|
50
|
+
"label": "Browser",
|
|
51
|
+
"family": "agent",
|
|
52
|
+
"summary": "A browser reached through an MCP server Nullius starts.",
|
|
53
|
+
"kit": None,
|
|
54
|
+
},
|
|
55
|
+
"cap.memory/v1": {
|
|
56
|
+
"label": "Memory",
|
|
57
|
+
"family": "capability",
|
|
58
|
+
"summary": "add {session} then recall {question} -> {memory}: what it kept that bears on the question.",
|
|
59
|
+
"kit": "cap-memory-v1",
|
|
60
|
+
},
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def family(port: str) -> str:
|
|
65
|
+
"""``agent`` or ``capability``: the side of a pairing a port belongs to."""
|
|
66
|
+
return "capability" if names.port(port).startswith("cap.") else port.split(".", 1)[0]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def known(port: str) -> bool:
|
|
70
|
+
return names.port(port) in PORTS
|