nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""A Harbor benchmark added by hand: read from Harbor, then written as a catalogue folder.
|
|
2
|
+
|
|
3
|
+
The Benchmarks page's *Add a Harbor benchmark* form uses two steps:
|
|
4
|
+
|
|
5
|
+
- :func:`read` asks Harbor about a dataset, on the hub (``org/name``) or in
|
|
6
|
+
the registry (``name@version``): how many tasks, its description and
|
|
7
|
+
version, and the limits one of its tasks declares in its ``task.toml`` -
|
|
8
|
+
the environment build, the agent and the verifier. Harbor applies those
|
|
9
|
+
limits to every trial, so they are what the builder's timeouts start
|
|
10
|
+
from. Agent setup is the agent's own default, Harbor's 360 s.
|
|
11
|
+
- :func:`build` turns the form into a ``benchmark.json`` and :func:`save`
|
|
12
|
+
writes it to ``benchmarks/<key>/``, after the catalogue's own checks
|
|
13
|
+
(:func:`catalog.validate`) pass on the catalogue with it added. An
|
|
14
|
+
existing folder is never written over.
|
|
15
|
+
- An entry the form can show (:func:`not_editable`) is edited the same way:
|
|
16
|
+
:func:`values_of` fills the form, :func:`merged` keeps whatever the form
|
|
17
|
+
does not show, and :func:`update` writes it back over itself.
|
|
18
|
+
|
|
19
|
+
Only Harbor routes: a Nullius route is brought in by ``/import-benchmark``,
|
|
20
|
+
then ``/import-standard``. Nothing here imports Flask; Harbor is reached through a ``fetch``
|
|
21
|
+
seam.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import tempfile
|
|
29
|
+
import tomllib
|
|
30
|
+
from dataclasses import replace
|
|
31
|
+
from urllib import error, parse, request
|
|
32
|
+
|
|
33
|
+
from . import catalog, storage
|
|
34
|
+
|
|
35
|
+
HUB = "https://hub.harborframework.com"
|
|
36
|
+
REGISTRY = "https://raw.githubusercontent.com/laude-institute/harbor/main/registry.json"
|
|
37
|
+
AGENT_SETUP_SEC = 360 # Harbor's default for an agent's setup; the task does not declare it
|
|
38
|
+
KEY = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
|
39
|
+
HUB_REF = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
|
40
|
+
REGISTRY_REF = re.compile(r"^[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+$")
|
|
41
|
+
SOURCES = {"hub": "Harbor hub", "registry": "Harbor registry"}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ReadError(Exception):
|
|
45
|
+
"""Harbor could not be read for that dataset; the message says why."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _fetch(url: str, timeout: float = 20.0) -> bytes:
|
|
49
|
+
req = request.Request(url, headers={"User-Agent": "nullius", "Accept": "application/json, text/plain, */*"})
|
|
50
|
+
with request.urlopen(req, timeout=timeout) as r:
|
|
51
|
+
return r.read()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _get(fetch, url: str, what: str) -> bytes:
|
|
55
|
+
try:
|
|
56
|
+
return fetch(url)
|
|
57
|
+
except error.HTTPError as e:
|
|
58
|
+
raise ReadError(f"{what}: Harbor answered {e.code}") from e
|
|
59
|
+
except (error.URLError, OSError, TimeoutError) as e:
|
|
60
|
+
raise ReadError(f"{what}: Harbor could not be reached ({getattr(e, 'reason', e)})") from e
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def timeouts_of(toml_text: str) -> dict:
|
|
64
|
+
"""The four phase limits, in seconds, from a task's ``task.toml``; a
|
|
65
|
+
limit the task leaves out is Harbor's default."""
|
|
66
|
+
try:
|
|
67
|
+
t = tomllib.loads(toml_text)
|
|
68
|
+
except tomllib.TOMLDecodeError as e:
|
|
69
|
+
raise ReadError(f"the task's task.toml does not parse: {e}") from e
|
|
70
|
+
sec = lambda v, d: int(round(float(v))) if isinstance(v, (int, float)) and v > 0 else d
|
|
71
|
+
return {"env_setup": sec((t.get("environment") or {}).get("build_timeout_sec"), 600),
|
|
72
|
+
"agent_setup": AGENT_SETUP_SEC,
|
|
73
|
+
"agent_run": sec((t.get("agent") or {}).get("timeout_sec"), 3600),
|
|
74
|
+
"verify": sec((t.get("verifier") or {}).get("timeout_sec"), 600)}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def read(source: str, ref: str, fetch=_fetch) -> dict:
|
|
78
|
+
"""What Harbor says of a dataset: ``{source, ref, tasks, version,
|
|
79
|
+
description, task, timeouts, key, title}``. ``key`` and ``title`` are
|
|
80
|
+
suggestions from the name."""
|
|
81
|
+
ref = (ref or "").strip()
|
|
82
|
+
if source == "hub":
|
|
83
|
+
if not HUB_REF.match(ref):
|
|
84
|
+
raise ReadError("a hub dataset is org/name, as on hub.harborframework.com")
|
|
85
|
+
org, name = ref.split("/", 1)
|
|
86
|
+
listing = json.loads(_get(fetch, f"{HUB}/api/datasets/{parse.quote(org)}/{parse.quote(name)}/latest/tasks", f"the hub dataset {ref}"))
|
|
87
|
+
items = listing.get("items") or []
|
|
88
|
+
if not items:
|
|
89
|
+
raise ReadError(f"the hub dataset {ref} has no tasks")
|
|
90
|
+
first = items[0]
|
|
91
|
+
toml = _get(fetch, f"{HUB}/api/tasks/{parse.quote(first['org_name'])}/{parse.quote(first['task_name'])}/latest/files/task.toml",
|
|
92
|
+
f"the task {first['task_name']}").decode("utf-8", "replace")
|
|
93
|
+
version, description = "", ""
|
|
94
|
+
try: # the page's own summary: "Tasks: 731 · Version: 2.0.0 · …"
|
|
95
|
+
page = _get(fetch, f"{HUB}/datasets/{parse.quote(org)}/{parse.quote(name)}", "the dataset page").decode("utf-8", "replace")
|
|
96
|
+
meta = re.search(r'<meta name="description" content="([^"]*)"', page)
|
|
97
|
+
if meta:
|
|
98
|
+
parts = [p.strip() for p in meta.group(1).split("·")]
|
|
99
|
+
version = next((p.split(":", 1)[1].strip() for p in parts if p.startswith("Version:")), "")
|
|
100
|
+
description = next((p for p in parts if not p.startswith(("Tasks:", "Version:"))), "")
|
|
101
|
+
if description == "A collection of evaluation tasks on Harbor.": # the hub's words for a dataset that gave none
|
|
102
|
+
description = ""
|
|
103
|
+
except ReadError:
|
|
104
|
+
pass
|
|
105
|
+
n, task, base = int(listing.get("total") or len(items)), first["task_name"], name
|
|
106
|
+
elif source == "registry":
|
|
107
|
+
if not REGISTRY_REF.match(ref):
|
|
108
|
+
raise ReadError("a registry dataset is name@version, as Harbor's registry lists it")
|
|
109
|
+
name, version = ref.split("@", 1)
|
|
110
|
+
entries = json.loads(_get(fetch, REGISTRY, "Harbor's registry"))
|
|
111
|
+
found = next((e for e in entries if e.get("name") == name and str(e.get("version")) == version), None)
|
|
112
|
+
if found is None:
|
|
113
|
+
versions = sorted(str(e.get("version")) for e in entries if e.get("name") == name)
|
|
114
|
+
raise ReadError(f"{ref} is not in Harbor's registry" + (f"; its versions: {', '.join(versions)}" if versions else ""))
|
|
115
|
+
tasks = found.get("tasks") or []
|
|
116
|
+
if not tasks:
|
|
117
|
+
raise ReadError(f"{ref} has no tasks in Harbor's registry")
|
|
118
|
+
first = tasks[0]
|
|
119
|
+
m = re.match(r"^https://github\.com/([^/]+/[^/.]+)(?:\.git)?$", first.get("git_url") or "")
|
|
120
|
+
if not m:
|
|
121
|
+
raise ReadError(f"{ref}'s tasks live at {first.get('git_url')!r}, which is not a GitHub repository this can read")
|
|
122
|
+
toml = _get(fetch, f"https://raw.githubusercontent.com/{m.group(1)}/{first.get('git_commit_id') or 'main'}/{first['path']}/task.toml",
|
|
123
|
+
f"the task {first.get('name')}").decode("utf-8", "replace")
|
|
124
|
+
n, task, base, description = len(tasks), first.get("name") or first["path"], name, found.get("description") or ""
|
|
125
|
+
else:
|
|
126
|
+
raise ReadError("the source is the hub or the registry")
|
|
127
|
+
key = re.sub(r"[^a-z0-9]+", "-", base.lower()).strip("-")[:63] or "benchmark"
|
|
128
|
+
return {"source": source, "ref": ref, "tasks": n, "version": version, "description": description, "task": task,
|
|
129
|
+
"timeouts": timeouts_of(toml), "key": key, "title": base.replace("-", " ").replace("_", " ").strip().title()}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _lines(text) -> list:
|
|
133
|
+
return [x.strip() for x in str(text or "").splitlines() if x.strip()]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def build(form: dict) -> dict:
|
|
137
|
+
"""The ``benchmark.json`` the form describes, in the catalogue's field
|
|
138
|
+
order. Fields are trimmed; empty optional ones are left out."""
|
|
139
|
+
g = lambda k: str(form.get(k) or "").strip()
|
|
140
|
+
source, ref = g("source"), g("ref")
|
|
141
|
+
timeouts = {}
|
|
142
|
+
for phase in ("env_setup", "agent_setup", "agent_run", "verify"):
|
|
143
|
+
try:
|
|
144
|
+
timeouts[phase] = int(float(g(f"timeout_{phase}")))
|
|
145
|
+
except ValueError:
|
|
146
|
+
timeouts = {}
|
|
147
|
+
break
|
|
148
|
+
dataset = {"key": g("dataset_key") or g("key"), "title": g("dataset_title") or g("title"),
|
|
149
|
+
"tasks": int(g("tasks")) if g("tasks").isdigit() else 0, "official": g("official"),
|
|
150
|
+
"source": SOURCES.get(source, source),
|
|
151
|
+
# the builder runs `harbor run -d <registry>`: a hub dataset names itself there too
|
|
152
|
+
**({"hub": ref, "registry": ref} if source == "hub" else {"registry": ref}),
|
|
153
|
+
"verifier": g("verifier")}
|
|
154
|
+
entry = {"key": g("key"), "title": g("title"), "category": g("category_new") or g("category"),
|
|
155
|
+
"domain": g("domain"), "scoring": g("scoring"), "harness": ["harbor"],
|
|
156
|
+
"origin": g("origin") or "official"}
|
|
157
|
+
if _lines(form.get("upstream_limitations")):
|
|
158
|
+
entry["upstream_limitations"] = _lines(form.get("upstream_limitations"))
|
|
159
|
+
entry["limitations"] = _lines(form.get("limitations"))
|
|
160
|
+
entry["datasets"] = [dataset]
|
|
161
|
+
if timeouts:
|
|
162
|
+
entry["timeouts"] = timeouts
|
|
163
|
+
if g("citation"):
|
|
164
|
+
entry["citation"] = g("citation")
|
|
165
|
+
if g("citation_source"):
|
|
166
|
+
entry["citation_source"] = g("citation_source")
|
|
167
|
+
return entry
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def problems(entry: dict, root: str | None = None, replacing: str = "", dataset: int = 0) -> list:
|
|
171
|
+
"""What stops the entry from being saved: the form's own rules, then the
|
|
172
|
+
catalogue's checks run on the catalogue with this entry added - or, when
|
|
173
|
+
``replacing`` names the entry being edited, with it in that one's place."""
|
|
174
|
+
bad = []
|
|
175
|
+
key = entry.get("key", "")
|
|
176
|
+
if not KEY.fullmatch(key):
|
|
177
|
+
bad.append("key: lowercase letters, digits and dashes, starting with a letter or digit")
|
|
178
|
+
elif key != replacing and any(os.path.lexists(os.path.join(base, catalog.BENCHMARKS, key))
|
|
179
|
+
for base in storage.roots(catalog.ROOT, root)):
|
|
180
|
+
bad.append(f"key: benchmarks/{key}/ exists already; an entry is never written over")
|
|
181
|
+
if replacing and not KEY.fullmatch(replacing):
|
|
182
|
+
bad.append("key: the entry being edited must have a valid key")
|
|
183
|
+
elif replacing and replacing != key and root is None and os.path.isdir(os.path.join(catalog.ROOT, catalog.BENCHMARKS, replacing)):
|
|
184
|
+
bad.append("key: a bundled entry keeps its key; edit its user copy under the same key")
|
|
185
|
+
for field, label in (("title", "title"), ("category", "category"), ("domain", "domain"), ("scoring", "scoring")):
|
|
186
|
+
if not entry.get(field):
|
|
187
|
+
bad.append(f"{label}: needed")
|
|
188
|
+
d = (entry.get("datasets") or [{}])[dataset] # the Harbor dataset the form wrote: the first, or a route's, the last
|
|
189
|
+
ref = d.get("hub") or d.get("registry") or ""
|
|
190
|
+
if not (HUB_REF.match(ref) if d.get("hub") else REGISTRY_REF.match(ref)):
|
|
191
|
+
bad.append("dataset: org/name on the hub, name@version in the registry")
|
|
192
|
+
if not d.get("tasks"):
|
|
193
|
+
bad.append("tasks: how many the dataset holds")
|
|
194
|
+
if not d.get("verifier"):
|
|
195
|
+
bad.append("verifier: how a task is scored, in a sentence")
|
|
196
|
+
if bad:
|
|
197
|
+
return bad
|
|
198
|
+
cat = catalog.load(root)
|
|
199
|
+
before = catalog.validate(cat)
|
|
200
|
+
if replacing:
|
|
201
|
+
cat = replace(cat, benchmarks=[b for b in cat.benchmarks if b.key != replacing],
|
|
202
|
+
datasets=[x for x in cat.datasets if x.benchmark != replacing])
|
|
203
|
+
new_b = catalog._pick(catalog.Benchmark, {**entry, "datasets": [x.get("key") for x in entry["datasets"]]})
|
|
204
|
+
new_d = [catalog._pick(catalog.Dataset, {**x, "benchmark": key}) for x in entry["datasets"]]
|
|
205
|
+
added = replace(cat, benchmarks=cat.benchmarks + [new_b], datasets=cat.datasets + new_d)
|
|
206
|
+
collisions = {d.key for d in new_d} & {d.key for d in cat.datasets}
|
|
207
|
+
# A pre-existing collision must not permit a third entry to claim the
|
|
208
|
+
# same dataset merely because validate() already reported that key.
|
|
209
|
+
relevant = {f"dataset key {k!r} used twice: a dataset key is unique across the catalogue" for k in collisions}
|
|
210
|
+
return [p for p in catalog.validate(added) if p not in before or p in relevant]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _write(path: str, entry: dict) -> None:
|
|
214
|
+
temporary = None
|
|
215
|
+
try:
|
|
216
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=os.path.dirname(path),
|
|
217
|
+
prefix=".entry-", delete=False) as fh:
|
|
218
|
+
temporary = fh.name
|
|
219
|
+
json.dump(entry, fh, indent=2, ensure_ascii=False)
|
|
220
|
+
fh.write("\n")
|
|
221
|
+
os.replace(temporary, path)
|
|
222
|
+
finally:
|
|
223
|
+
if temporary is not None and os.path.exists(temporary):
|
|
224
|
+
os.unlink(temporary)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def save(entry: dict, root: str | None = None) -> str:
|
|
228
|
+
"""Write ``benchmarks/<key>/benchmark.json``; the path written. Refuses,
|
|
229
|
+
with what is wrong, anything :func:`problems` names."""
|
|
230
|
+
bad = problems(entry, root)
|
|
231
|
+
if bad:
|
|
232
|
+
raise ValueError("; ".join(bad))
|
|
233
|
+
folder = catalog.writable_dir(entry["key"], root)
|
|
234
|
+
os.makedirs(folder)
|
|
235
|
+
path = os.path.join(folder, catalog.ENTRY)
|
|
236
|
+
_write(path, entry)
|
|
237
|
+
return path
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# what the form shows of an entry, and so what a save of it replaces; anything
|
|
242
|
+
# else an entry holds (its report block, a branch, notes)
|
|
243
|
+
# is kept as it is. status, status_note and experiment are an older entry's
|
|
244
|
+
# words for its origin, and todo its open work, no longer kept: a save drops them
|
|
245
|
+
FORM_FIELDS = ("title", "category", "domain", "scoring", "harness", "status", "status_note", "origin",
|
|
246
|
+
"experiment", "upstream_limitations", "limitations", "todo", "timeouts", "citation", "citation_source")
|
|
247
|
+
DATASET_FIELDS = ("key", "title", "tasks", "official", "source", "hub", "registry", "verifier")
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def entry_of(key: str, root: str | None = None) -> dict | None:
|
|
251
|
+
"""The entry as written in ``benchmarks/<key>/benchmark.json``, or None."""
|
|
252
|
+
folder = catalog.entry_dir(key, root)
|
|
253
|
+
if folder is None:
|
|
254
|
+
return None
|
|
255
|
+
path = os.path.join(folder, catalog.ENTRY)
|
|
256
|
+
try:
|
|
257
|
+
with open(path, encoding="utf-8") as fh:
|
|
258
|
+
return json.load(fh)
|
|
259
|
+
except (OSError, ValueError):
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def not_editable(entry: dict) -> str:
|
|
264
|
+
"""Why the form cannot edit this entry, or ''. It edits a Harbor entry
|
|
265
|
+
with one dataset, on the hub or in the registry; an entry that also runs
|
|
266
|
+
on the Nullius standard is edited in its files, with the /import-standard skill."""
|
|
267
|
+
if entry.get("harness") != ["harbor"]:
|
|
268
|
+
return "an entry on the Nullius standard is edited in its files (the /import-standard skill)"
|
|
269
|
+
ds = entry.get("datasets") or []
|
|
270
|
+
if len(ds) != 1:
|
|
271
|
+
return "an entry with several datasets is edited in its benchmark.json"
|
|
272
|
+
if not (ds[0].get("hub") or ds[0].get("registry")):
|
|
273
|
+
return "its dataset is neither on the hub nor in the registry"
|
|
274
|
+
return ""
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def values_of(entry: dict) -> dict:
|
|
278
|
+
"""The form's fields, filled from an entry."""
|
|
279
|
+
d = (entry.get("datasets") or [{}])[0]
|
|
280
|
+
hub = d.get("hub") or ""
|
|
281
|
+
t = entry.get("timeouts") or {}
|
|
282
|
+
values = {"key": entry.get("key", ""), "title": entry.get("title", ""), "category": entry.get("category", ""),
|
|
283
|
+
"domain": entry.get("domain", ""), "scoring": entry.get("scoring", ""),
|
|
284
|
+
"origin": "official" if entry.get("experiment") or entry.get("origin") == "experiment"
|
|
285
|
+
else entry.get("origin", "official"), "upstream_limitations": "\n".join(entry.get("upstream_limitations") or []),
|
|
286
|
+
"limitations": "\n".join(entry.get("limitations") or []),
|
|
287
|
+
"citation": entry.get("citation", ""),
|
|
288
|
+
"citation_source": entry.get("citation_source", ""),
|
|
289
|
+
"source": "hub" if hub else "registry", "ref": hub or d.get("registry", ""),
|
|
290
|
+
"tasks": str(d.get("tasks") or ""), "verifier": d.get("verifier", ""), "official": d.get("official", ""),
|
|
291
|
+
"dataset_key": d.get("key", ""), "dataset_title": d.get("title", "")}
|
|
292
|
+
values.update({f"timeout_{k}": str(t[k]) for k in ("env_setup", "agent_setup", "agent_run", "verify") if k in t})
|
|
293
|
+
return values
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def merged(old: dict, new: dict) -> dict:
|
|
297
|
+
"""``old`` with what the form owns taken from ``new``: a field the form
|
|
298
|
+
left empty is removed, everything the form does not show is kept."""
|
|
299
|
+
out = {k: v for k, v in old.items() if k not in FORM_FIELDS}
|
|
300
|
+
for k in FORM_FIELDS:
|
|
301
|
+
if k in new:
|
|
302
|
+
out[k] = new[k]
|
|
303
|
+
old_d, new_d = (old.get("datasets") or [{}])[0], new["datasets"][0]
|
|
304
|
+
d = {k: v for k, v in old_d.items() if k not in DATASET_FIELDS}
|
|
305
|
+
d.update({k: v for k, v in new_d.items() if k in DATASET_FIELDS})
|
|
306
|
+
out["datasets"] = [d] + list((old.get("datasets") or [])[1:])
|
|
307
|
+
return out
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def update(entry: dict, root: str | None = None, was: str = "", dataset: int = 0) -> str:
|
|
311
|
+
"""Write an edited entry over its own ``benchmark.json``; the path
|
|
312
|
+
written. A bundled entry is first copied in full to the user catalogue.
|
|
313
|
+
``was`` is its key before the edit: for a user-only entry a new key moves
|
|
314
|
+
the folder, with everything in it, to the new name. Refuses, with what
|
|
315
|
+
is wrong, anything :func:`problems` names."""
|
|
316
|
+
was = was or entry.get("key", "")
|
|
317
|
+
bad = problems(entry, root, replacing=was, dataset=dataset)
|
|
318
|
+
if bad:
|
|
319
|
+
raise ValueError("; ".join(bad))
|
|
320
|
+
selected = catalog.entry_dir(was, root)
|
|
321
|
+
if selected is None or not os.path.isfile(os.path.join(selected, catalog.ENTRY)):
|
|
322
|
+
raise ValueError(f"benchmarks/{was}/ has no entry to edit")
|
|
323
|
+
old = catalog.writable_dir(was, root)
|
|
324
|
+
folder = os.path.join(os.path.dirname(old), entry["key"])
|
|
325
|
+
storage.ensure_writable(folder, catalog.ROOT)
|
|
326
|
+
if folder != old:
|
|
327
|
+
os.rename(old, folder) # problems() said the new name is free
|
|
328
|
+
path = os.path.join(folder, catalog.ENTRY)
|
|
329
|
+
_write(path, entry)
|
|
330
|
+
return path
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def delete(key: str, root: str | None = None) -> str:
|
|
334
|
+
"""Remove a user entry folder; removing an override restores the bundle.
|
|
335
|
+
Bundled-only entries cannot be deleted. Only an entry the form edits
|
|
336
|
+
(:func:`not_editable`) is removed here."""
|
|
337
|
+
import shutil
|
|
338
|
+
entry = entry_of(key, root)
|
|
339
|
+
if entry is None or not KEY.fullmatch(key):
|
|
340
|
+
raise ValueError(f"benchmarks/{key}/ has no entry")
|
|
341
|
+
why = not_editable(entry)
|
|
342
|
+
if why:
|
|
343
|
+
raise ValueError(f"not removed here: {why}")
|
|
344
|
+
folder = os.path.join(storage.writable_root(catalog.ROOT, root), catalog.BENCHMARKS, key)
|
|
345
|
+
storage.ensure_writable(folder, catalog.ROOT)
|
|
346
|
+
if not os.path.isdir(folder):
|
|
347
|
+
raise ValueError("the bundled catalogue is read-only; only user entries can be removed")
|
|
348
|
+
shutil.rmtree(folder)
|
|
349
|
+
return folder
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def harbor_route(old: dict, values: dict) -> dict:
|
|
353
|
+
"""``old`` - a benchmark run on the Nullius standard - with a Harbor route
|
|
354
|
+
added: the form's dataset appended to its datasets, ``harbor`` to its
|
|
355
|
+
harnesses, the limits its tasks declare when it had none. Everything
|
|
356
|
+
else the entry says stays: it is one benchmark, run either way, and one
|
|
357
|
+
card. The dataset's key is the form's, else ``<key>-harbor``."""
|
|
358
|
+
built = build(dict(values, key=old["key"], title=values.get("dataset_title") or old.get("title", "")))
|
|
359
|
+
d = built["datasets"][0]
|
|
360
|
+
d["key"] = (values.get("dataset_key") or "").strip() or f"{old['key']}-harbor"
|
|
361
|
+
d["title"] = (values.get("dataset_title") or "").strip() or f"{old.get('title', old['key'])} on Harbor"
|
|
362
|
+
out = dict(old)
|
|
363
|
+
out["harness"] = list(old.get("harness") or []) + ([] if "harbor" in (old.get("harness") or []) else ["harbor"])
|
|
364
|
+
out["datasets"] = list(old.get("datasets") or []) + [d]
|
|
365
|
+
if built.get("timeouts") and not old.get("timeouts"):
|
|
366
|
+
out["timeouts"] = built["timeouts"]
|
|
367
|
+
return out
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Where a dataset's data comes from, one row for each way the benchmark runs.
|
|
2
|
+
|
|
3
|
+
A benchmark that runs with Harbor and the Nullius standard reads its data two
|
|
4
|
+
ways: Harbor from its registry or hub, the standard from the upstream it pins
|
|
5
|
+
(a repository at a commit, or a file at a revision) or from the Hugging Face
|
|
6
|
+
dataset its driver downloads (``data-source.json``: ``repository`` and
|
|
7
|
+
``revision``). Each row names the source, the pin and who fetches it - always
|
|
8
|
+
the harness, on a run, never the person - and the files a definition
|
|
9
|
+
checks by their sha256. Nothing here reads the network: it reads the
|
|
10
|
+
catalogue entry and the definitions beside it.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
from typing import List, Optional
|
|
18
|
+
|
|
19
|
+
_GITHUB = re.compile(r"^https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$")
|
|
20
|
+
_HF = re.compile(r"^https://huggingface\.co/datasets/([^/]+/[^/]+)/resolve/([^/]+)/")
|
|
21
|
+
_HF_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
|
22
|
+
_REVISION = re.compile(r"^[^/\s]+$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _size(n) -> str:
|
|
26
|
+
if not isinstance(n, int) or n <= 0:
|
|
27
|
+
return ""
|
|
28
|
+
for unit, scale in (("GB", 10 ** 9), ("MB", 10 ** 6), ("kB", 10 ** 3)):
|
|
29
|
+
if n >= scale:
|
|
30
|
+
return f"{n / scale:.1f} {unit}".replace(".0 ", " ")
|
|
31
|
+
return f"{n} B"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _repo(upstream: dict) -> dict:
|
|
35
|
+
"""The upstream repository as a row's source: GitHub's owner/repo, linked at
|
|
36
|
+
the pinned commit; its release tag, when it has one, beside the commit."""
|
|
37
|
+
url, commit = upstream.get("repository", ""), upstream.get("tested", "")
|
|
38
|
+
m = _GITHUB.match(url)
|
|
39
|
+
pin = " · ".join(x for x in (upstream.get("tag"), commit[:7]) if x)
|
|
40
|
+
return {"from": "GitHub" if m else "Repository", "name": m.group(1) if m else url,
|
|
41
|
+
"href": f"{url.rstrip('/')}/tree/{commit}" if m and commit else url, "pin": pin}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _files(defined: List[dict]) -> List[dict]:
|
|
45
|
+
"""The files a definition's datasets name, each with its size when known."""
|
|
46
|
+
out = []
|
|
47
|
+
for d in defined:
|
|
48
|
+
where = d.get("path") or d.get("url") or ""
|
|
49
|
+
if where:
|
|
50
|
+
out.append({"name": d.get("path") or os.path.basename(where), "size": _size(d.get("bytes")),
|
|
51
|
+
"checked": bool(d.get("sha256"))})
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _data_source(here) -> Optional[tuple]:
|
|
56
|
+
"""The Hugging Face dataset a driver downloads itself, ``data-source.json``
|
|
57
|
+
beside its definition: (repository, revision), or None."""
|
|
58
|
+
if not here:
|
|
59
|
+
return None
|
|
60
|
+
try:
|
|
61
|
+
with open(os.path.join(here, "data-source.json")) as f:
|
|
62
|
+
src = json.load(f)
|
|
63
|
+
repo, rev = src["repository"], src["revision"]
|
|
64
|
+
except (OSError, ValueError, TypeError, KeyError):
|
|
65
|
+
return None
|
|
66
|
+
ok = isinstance(repo, str) and _HF_REPO.match(repo) and isinstance(rev, str) and _REVISION.match(rev)
|
|
67
|
+
return (repo, rev) if ok else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _own(upstream: dict, defined: List[dict], who: str, source: Optional[tuple] = None) -> dict:
|
|
71
|
+
"""A row for a harness that reads the upstream itself: from Hugging Face when
|
|
72
|
+
its driver names a dataset there (``source``) or every file it names is
|
|
73
|
+
there, else from the repository it clones."""
|
|
74
|
+
hf = [_HF.match(d.get("url") or "") for d in defined]
|
|
75
|
+
if source or (defined and all(hf)):
|
|
76
|
+
name, rev = source or (hf[0].group(1), hf[0].group(2))
|
|
77
|
+
row = {"from": "Hugging Face", "name": name, "href": f"https://huggingface.co/datasets/{name}/tree/{rev}",
|
|
78
|
+
"pin": rev[:7], "by": f"{who} downloads it on the first run and keeps it"}
|
|
79
|
+
else:
|
|
80
|
+
row = dict(_repo(upstream), by=f"{who} fetches the repository at this commit on the first run and keeps it")
|
|
81
|
+
row["files"] = _files(defined)
|
|
82
|
+
return row
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _matching(defined: List[dict], key: str, only: bool) -> List[dict]:
|
|
86
|
+
"""The datasets of a definition that are this catalogue dataset: by id
|
|
87
|
+
(``-`` and ``_`` alike), or all of them when the entry has a single dataset."""
|
|
88
|
+
norm = key.replace("-", "_")
|
|
89
|
+
hit = [d for d in defined if str(d.get("id", "")).replace("-", "_") == norm]
|
|
90
|
+
return hit or (list(defined) if only else [])
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def of(dataset, harness: List[str], standard: Optional[dict] = None, only: bool = True) -> List[dict]:
|
|
94
|
+
"""One row per way ``dataset`` runs, in the order the page names them
|
|
95
|
+
(Nullius, Harbor): ``way``, ``label``, ``from``, ``name``, ``href``,
|
|
96
|
+
``pin``, ``by`` and ``files``. ``standard`` is the raw definition beside
|
|
97
|
+
the entry; ``only`` says the entry has one dataset."""
|
|
98
|
+
rows = []
|
|
99
|
+
if standard:
|
|
100
|
+
rows.append(dict(_own(standard.get("upstream") or {}, _matching(standard.get("datasets") or [], dataset.key, only), "Nullius",
|
|
101
|
+
_data_source(standard.get("_here"))),
|
|
102
|
+
way="standard", label="Nullius"))
|
|
103
|
+
if "harbor" in harness and (dataset.hub or dataset.registry):
|
|
104
|
+
rows.append({"way": "harbor", "label": "Harbor", "from": "Harbor hub" if dataset.hub else "Harbor registry",
|
|
105
|
+
"name": dataset.hub or dataset.registry, "href": "", "pin": "",
|
|
106
|
+
"by": "Harbor downloads it when a run needs it", "files": []})
|
|
107
|
+
return rows
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Whole-entry catalogue overlays, shared by the catalogue and the standard.
|
|
2
|
+
|
|
3
|
+
An explicit root is a single isolated tree. Otherwise the user's folder wins
|
|
4
|
+
over the bundled folder of the same name, including when its entry is invalid.
|
|
5
|
+
We never combine files from different copies of an entry.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
import stat
|
|
12
|
+
import tempfile
|
|
13
|
+
|
|
14
|
+
from nullius import paths
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def roots(bundled: str, root: str | None = None) -> tuple:
|
|
18
|
+
if root is not None:
|
|
19
|
+
return (os.fspath(root),)
|
|
20
|
+
user = paths.catalogue_dir()
|
|
21
|
+
return (bundled,) if os.path.realpath(user) == os.path.realpath(bundled) else (bundled, user)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def folders(group: str, bundled: str, root: str | None = None) -> dict:
|
|
25
|
+
"""Selected entry folders, sorted by key; an empty folder still wins."""
|
|
26
|
+
found = {}
|
|
27
|
+
for base in roots(bundled, root):
|
|
28
|
+
for directory in (("features", "capabilities") if group == "capabilities" else (group,)):
|
|
29
|
+
parent = os.path.join(base, directory)
|
|
30
|
+
try:
|
|
31
|
+
names = os.listdir(parent)
|
|
32
|
+
except FileNotFoundError:
|
|
33
|
+
continue
|
|
34
|
+
for name in sorted(names):
|
|
35
|
+
folder = os.path.join(parent, name)
|
|
36
|
+
if name.startswith((".", "_")) or not os.path.isdir(folder):
|
|
37
|
+
continue
|
|
38
|
+
if os.listdir(folder) == ["__pycache__"]:
|
|
39
|
+
continue
|
|
40
|
+
found[name] = folder
|
|
41
|
+
return dict(sorted(found.items()))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def entry_dir(group: str, key: str, bundled: str, root: str | None = None) -> str | None:
|
|
45
|
+
return folders(group, bundled, root).get(key)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def writable_root(bundled: str, root: str | None = None) -> str:
|
|
49
|
+
"""The destination, refusing paths that would mutate package resources."""
|
|
50
|
+
target = os.fspath(root) if root is not None else paths.catalogue_dir()
|
|
51
|
+
ensure_writable(target, bundled)
|
|
52
|
+
return target
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def ensure_writable(target: str, bundled: str) -> None:
|
|
56
|
+
bundle = os.path.realpath(bundled)
|
|
57
|
+
try:
|
|
58
|
+
common = os.path.commonpath((os.path.realpath(target), bundle))
|
|
59
|
+
except ValueError: # different drives on Windows cannot overlap
|
|
60
|
+
return
|
|
61
|
+
if common == bundle:
|
|
62
|
+
raise ValueError("the bundled catalogue is read-only; use the user catalogue")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def writable_dir(group: str, key: str, bundled: str, root: str | None = None) -> str:
|
|
66
|
+
"""Copy a selected bundled folder in full before returning its user copy.
|
|
67
|
+
|
|
68
|
+
An existing user folder is left intact. Staging keeps an interrupted copy
|
|
69
|
+
from shadowing a bundled entry with half of its files.
|
|
70
|
+
"""
|
|
71
|
+
if not key or key in (".", "..") or os.path.basename(key) != key:
|
|
72
|
+
raise ValueError("an entry key is a folder name")
|
|
73
|
+
target = os.path.join(writable_root(bundled, root), group, key)
|
|
74
|
+
ensure_writable(target, bundled)
|
|
75
|
+
source = entry_dir(group, key, bundled, root)
|
|
76
|
+
if source is None or os.path.realpath(source) == os.path.realpath(target):
|
|
77
|
+
return target
|
|
78
|
+
parent = os.path.dirname(target)
|
|
79
|
+
os.makedirs(parent, exist_ok=True)
|
|
80
|
+
with tempfile.TemporaryDirectory(prefix=".catalogue-", dir=parent) as stage:
|
|
81
|
+
copied = os.path.join(stage, key)
|
|
82
|
+
shutil.copytree(source, copied)
|
|
83
|
+
# Package resources may be read-only. The copy belongs to its user.
|
|
84
|
+
for folder, _, files in os.walk(copied):
|
|
85
|
+
os.chmod(folder, os.stat(folder).st_mode | stat.S_IWUSR)
|
|
86
|
+
for name in files:
|
|
87
|
+
path = os.path.join(folder, name)
|
|
88
|
+
os.chmod(path, os.stat(path).st_mode | stat.S_IWUSR)
|
|
89
|
+
os.rename(copied, target)
|
|
90
|
+
return target
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _restore_targets(group: str, key: str, bundled: str) -> list[str]:
|
|
94
|
+
if group not in ("benchmarks", "agents", "capabilities"):
|
|
95
|
+
raise ValueError("unknown catalogue")
|
|
96
|
+
if not key or key in (".", "..") or "/" in key or "\\" in key:
|
|
97
|
+
raise ValueError("an entry key is a folder name")
|
|
98
|
+
if entry_dir(group, key, bundled, root=bundled) is None:
|
|
99
|
+
raise ValueError("this entry has no bundled copy")
|
|
100
|
+
user = writable_root(bundled)
|
|
101
|
+
groups = ("features", "capabilities") if group == "capabilities" else (group,)
|
|
102
|
+
targets = [os.path.join(user, directory, key) for directory in groups
|
|
103
|
+
if os.path.isdir(os.path.join(user, directory, key))]
|
|
104
|
+
if not targets:
|
|
105
|
+
raise ValueError("this entry has no user copy")
|
|
106
|
+
for target in targets:
|
|
107
|
+
ensure_writable(target, bundled)
|
|
108
|
+
return targets
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def can_restore(group: str, key: str, bundled: str) -> bool:
|
|
112
|
+
"""Whether removing the user override can reveal a bundled entry."""
|
|
113
|
+
try:
|
|
114
|
+
return bool(_restore_targets(group, key, bundled))
|
|
115
|
+
except (OSError, ValueError):
|
|
116
|
+
return False
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def use_bundled(group: str, key: str, bundled: str) -> None:
|
|
120
|
+
"""Remove complete user overrides, including a capability's legacy alias."""
|
|
121
|
+
for target in _restore_targets(group, key, bundled):
|
|
122
|
+
if os.path.islink(target):
|
|
123
|
+
os.unlink(target)
|
|
124
|
+
else:
|
|
125
|
+
shutil.rmtree(target)
|