nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/verifier.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""The verifier's own report, read as the benchmark's entry describes it.
|
|
2
|
+
|
|
3
|
+
Everything else this app reads is Harbor's and the same for every benchmark:
|
|
4
|
+
``result.json``, ``config.json``, ``exception_info``, ``trace.json``, the ATIF
|
|
5
|
+
trajectory and ``verifier_result.rewards.reward``. The verifier's report is the
|
|
6
|
+
exception - one benchmark writes a ``report.json``, another a CTRF
|
|
7
|
+
``ctrf.json`` - and its keys are the benchmark's. So no code here names them:
|
|
8
|
+
a benchmark's ``benchmark.json`` carries a ``report`` block, and this module
|
|
9
|
+
reads any report by it::
|
|
10
|
+
|
|
11
|
+
"report": {
|
|
12
|
+
"sources": [...], the dataset names a Harbor job records for it
|
|
13
|
+
"file": "verifier/report.json",
|
|
14
|
+
"required": true, no file: the verifier never ran (else the
|
|
15
|
+
reward file stands in for it)
|
|
16
|
+
"under_task": true, the document is keyed by the task's name
|
|
17
|
+
"succeeded": "resolved", a field, or "reward": Harbor's 1.0
|
|
18
|
+
"produced": "patch_exists", "produced_unless": "patch_is_None",
|
|
19
|
+
"usable": "patch_successfully_applied",
|
|
20
|
+
"gained": {"pass": [paths], "fail": [paths]}, lists of check names
|
|
21
|
+
"held": {...}, the same, or
|
|
22
|
+
"gained": {"tests": path, "name": "name", "status": "status",
|
|
23
|
+
"pass": "passed", "fail": "failed",
|
|
24
|
+
"count_pass": path, "count_fail": path}, a list of tests
|
|
25
|
+
"words": {"gained": "Fail → Pass", "gained_abbr", "gained_term",
|
|
26
|
+
"held", ..., "has_patch": true}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
A path is dotted (``tests_status.FAIL_TO_PASS.success``). *gained* are the
|
|
30
|
+
checks that had to start passing, *held* the ones that had to keep passing: a
|
|
31
|
+
benchmark with one pass or fail leaves ``held`` out. A benchmark with no block
|
|
32
|
+
still parses: the reward is all that is read, and its pages show fewer
|
|
33
|
+
columns. A field the report cannot answer is ``None``, never a plausible zero.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import json
|
|
39
|
+
import os
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Verdict:
|
|
45
|
+
"""What a verifier said, in terms no single benchmark owns.
|
|
46
|
+
|
|
47
|
+
`None` means *this benchmark does not report that*, and is distinct from
|
|
48
|
+
`False` / `0`, which mean it reported the absence.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
succeeded: bool | None = None
|
|
52
|
+
produced: bool | None = None # did the agent emit a product at all?
|
|
53
|
+
usable: bool | None = None # did that product apply / run?
|
|
54
|
+
gained_pass: int = 0
|
|
55
|
+
gained_fail: int = 0
|
|
56
|
+
held_pass: int = 0
|
|
57
|
+
held_fail: int = 0
|
|
58
|
+
gained_failures: tuple[str, ...] = ()
|
|
59
|
+
held_failures: tuple[str, ...] = ()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
UNREAD = Verdict()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Vocabulary:
|
|
67
|
+
"""The benchmark's own names for its checks on screen (``report.words``).
|
|
68
|
+
|
|
69
|
+
`None` means the benchmark has no such category, and the pages leave the
|
|
70
|
+
row or column out rather than dash it under a foreign label. `has_patch`
|
|
71
|
+
gates the "Patch applied" card the same way.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
gained: str | None = None # card label, e.g. "Fail → Pass" / "Tests"
|
|
75
|
+
gained_abbr: str = "" # table-column form; empty falls back to `gained`
|
|
76
|
+
gained_term: str = "" # the dataset's jargon name, when it has one
|
|
77
|
+
held: str | None = None
|
|
78
|
+
held_abbr: str = ""
|
|
79
|
+
held_term: str = ""
|
|
80
|
+
has_patch: bool = False
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# What a benchmark without a report block gets: no named categories, so the
|
|
84
|
+
# check columns and patch cards are simply not drawn.
|
|
85
|
+
GENERIC_VOCABULARY = Vocabulary()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def reward_is_success(reward: float | None) -> bool | None:
|
|
89
|
+
"""Harbor's convention: 1.0 is a solve. `None` in, `None` out."""
|
|
90
|
+
return None if reward is None else reward >= 1.0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _read_json(path: str):
|
|
94
|
+
try:
|
|
95
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
96
|
+
return json.load(fh)
|
|
97
|
+
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def reward_marker(trial_dir: str):
|
|
102
|
+
"""`{}` when the verifier left its reward file, `None` when it left nothing.
|
|
103
|
+
|
|
104
|
+
`verifier/reward.txt`, or a `verifier/reward.json` object with a numeric
|
|
105
|
+
`reward` (harbor-rewardkit): either is evidence the verifier ran. The empty
|
|
106
|
+
dict is deliberately not `None`: the verifier spoke and said no more than
|
|
107
|
+
the reward.
|
|
108
|
+
"""
|
|
109
|
+
verifier = os.path.join(trial_dir, "verifier")
|
|
110
|
+
if os.path.isfile(os.path.join(verifier, "reward.txt")):
|
|
111
|
+
return {}
|
|
112
|
+
doc = _read_json(os.path.join(verifier, "reward.json"))
|
|
113
|
+
if isinstance(doc, dict) and isinstance(doc.get("reward"), (int, float)) \
|
|
114
|
+
and not isinstance(doc.get("reward"), bool):
|
|
115
|
+
return {}
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_CACHE: dict = {}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _blocks() -> dict:
|
|
124
|
+
"""Every ``report`` block in the catalogue, by each source name it lists."""
|
|
125
|
+
from nullius.catalogue import catalog
|
|
126
|
+
sig = catalog.signature()
|
|
127
|
+
if _CACHE.get("sig") != sig:
|
|
128
|
+
table = {}
|
|
129
|
+
for b in catalog.load().benchmarks:
|
|
130
|
+
if b.report:
|
|
131
|
+
for name in b.report.get("sources") or [b.key]:
|
|
132
|
+
table[name.strip().lower()] = (b.key, b.report)
|
|
133
|
+
_CACHE.update(sig=sig, table=table)
|
|
134
|
+
return _CACHE["table"]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def block_for(source: str | None):
|
|
138
|
+
"""``(benchmark key, report block)`` for *source*, or ``None``."""
|
|
139
|
+
return _blocks().get((source or "").strip().lower())
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _dig(doc, path: str):
|
|
143
|
+
for part in path.split("."):
|
|
144
|
+
if not isinstance(doc, dict):
|
|
145
|
+
return None
|
|
146
|
+
doc = doc.get(part)
|
|
147
|
+
return doc
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _names(doc, paths) -> list:
|
|
151
|
+
out = []
|
|
152
|
+
for p in paths or []:
|
|
153
|
+
got = _dig(doc, p)
|
|
154
|
+
if isinstance(got, list):
|
|
155
|
+
out += [str(x) for x in got]
|
|
156
|
+
return out
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _checks(doc, spec) -> tuple[int, int, tuple]:
|
|
160
|
+
"""``(passed, failed, failing names)`` of one category."""
|
|
161
|
+
if not spec:
|
|
162
|
+
return 0, 0, ()
|
|
163
|
+
if "tests" in spec: # a list of test objects with a status
|
|
164
|
+
tests = [t for t in (_dig(doc, spec["tests"]) or []) if isinstance(t, dict)]
|
|
165
|
+
name, status = spec.get("name", "name"), spec.get("status", "status")
|
|
166
|
+
failing = tuple(str(t.get(name, "")) for t in tests if t.get(status) == spec.get("fail", "failed"))
|
|
167
|
+
# the producer's own counts when it gives them, as trace totals are taken
|
|
168
|
+
passed = _dig(doc, spec["count_pass"]) if spec.get("count_pass") else None
|
|
169
|
+
failed = _dig(doc, spec["count_fail"]) if spec.get("count_fail") else None
|
|
170
|
+
if passed is None and failed is None and tests:
|
|
171
|
+
passed = sum(1 for t in tests if t.get(status) == spec.get("pass", "passed"))
|
|
172
|
+
failed = len(failing)
|
|
173
|
+
return int(passed or 0), int(failed or 0), failing
|
|
174
|
+
ok, bad = _names(doc, spec.get("pass")), _names(doc, spec.get("fail"))
|
|
175
|
+
return len(ok), len(bad), tuple(bad)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _read_block(spec: dict, report, reward) -> Verdict:
|
|
179
|
+
if not isinstance(report, dict):
|
|
180
|
+
return Verdict()
|
|
181
|
+
how = spec.get("succeeded", "reward")
|
|
182
|
+
succeeded = reward_is_success(reward) if how == "reward" else bool(_dig(report, how))
|
|
183
|
+
produced = usable = None
|
|
184
|
+
if spec.get("produced"):
|
|
185
|
+
produced = bool(_dig(report, spec["produced"])) and not (
|
|
186
|
+
spec.get("produced_unless") and _dig(report, spec["produced_unless"]))
|
|
187
|
+
# only meaningful once something was produced
|
|
188
|
+
usable = bool(_dig(report, spec["usable"])) if spec.get("usable") and produced else None
|
|
189
|
+
gp, gf, gfail = _checks(report, spec.get("gained"))
|
|
190
|
+
hp, hf, hfail = _checks(report, spec.get("held"))
|
|
191
|
+
return Verdict(succeeded=succeeded, produced=produced, usable=usable,
|
|
192
|
+
gained_pass=gp, gained_fail=gf, held_pass=hp, held_fail=hf,
|
|
193
|
+
gained_failures=gfail, held_failures=hfail)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# --- what the rest of the app calls ------------------------------------------
|
|
197
|
+
|
|
198
|
+
def vocabulary_for(source: str | None) -> Vocabulary:
|
|
199
|
+
"""How *source* names its checks, or the empty vocabulary. Never raises."""
|
|
200
|
+
found = block_for(source)
|
|
201
|
+
if not found:
|
|
202
|
+
return GENERIC_VOCABULARY
|
|
203
|
+
words = found[1].get("words") or {}
|
|
204
|
+
return Vocabulary(**{k: v for k, v in words.items() if k in Vocabulary.__dataclass_fields__})
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def load(source: str | None, trial_dir: str, task_base: str):
|
|
208
|
+
"""Find *source*'s verifier report inside *trial_dir*.
|
|
209
|
+
|
|
210
|
+
Without a block: a ``verifier/report.json`` if there is one, else the
|
|
211
|
+
reward marker. Returns `None` when the verifier left nothing at all, which
|
|
212
|
+
`read()` refuses to score.
|
|
213
|
+
"""
|
|
214
|
+
found = block_for(source)
|
|
215
|
+
if not found:
|
|
216
|
+
report = _read_json(os.path.join(trial_dir, "verifier", "report.json"))
|
|
217
|
+
return report if isinstance(report, dict) else reward_marker(trial_dir)
|
|
218
|
+
spec = found[1]
|
|
219
|
+
report = _read_json(os.path.join(trial_dir, spec.get("file", "verifier/report.json")))
|
|
220
|
+
if not isinstance(report, dict):
|
|
221
|
+
return None if spec.get("required") else reward_marker(trial_dir)
|
|
222
|
+
if spec.get("under_task"):
|
|
223
|
+
inner = report.get(task_base)
|
|
224
|
+
if inner is None and len(report) == 1:
|
|
225
|
+
inner = next(iter(report.values()))
|
|
226
|
+
return inner
|
|
227
|
+
return report
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def read(source: str | None, report, reward: float | None) -> Verdict:
|
|
231
|
+
"""Turn a verifier report into a `Verdict`, however little is understood.
|
|
232
|
+
|
|
233
|
+
No report means the verifier never spoke, which is not the same as it
|
|
234
|
+
saying no — so `succeeded` stays `None` and the reward is *not* consulted.
|
|
235
|
+
"""
|
|
236
|
+
if report is None:
|
|
237
|
+
return UNREAD
|
|
238
|
+
found = block_for(source)
|
|
239
|
+
if not found:
|
|
240
|
+
return Verdict(succeeded=reward_is_success(reward))
|
|
241
|
+
return _read_block(found[1], report, reward)
|
nullius/sandbox.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""The Settings page's Sandbox: the container backend's engine on this machine,
|
|
2
|
+
docker or podman (``container.DOCKER``), and its Docker Hub login under Data
|
|
3
|
+
providers.
|
|
4
|
+
|
|
5
|
+
Two things, each something a user acts on:
|
|
6
|
+
|
|
7
|
+
- **Engine**: whether it answers, and its name and version (``container.unusable``).
|
|
8
|
+
- **Registry login**: Docker Hub limits anonymous pulls, which stops a
|
|
9
|
+
benchmark with an image per unit part-way. The username and an access token
|
|
10
|
+
go to ``docker login --password-stdin`` (``podman login ... docker.io``)
|
|
11
|
+
once; the engine keeps them in its own store (``~/.docker/config.json`` or
|
|
12
|
+
the credential helper it names; podman's ``auth.json``), the same as a
|
|
13
|
+
login typed in a terminal. Nullius stores neither, and reads back only the
|
|
14
|
+
username.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import base64
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import subprocess
|
|
23
|
+
from typing import List, Optional
|
|
24
|
+
|
|
25
|
+
from nullius.standard import container
|
|
26
|
+
|
|
27
|
+
REGISTRY = "https://index.docker.io/v1/" # Docker Hub, as docker names it in its config
|
|
28
|
+
HUB = "docker.io" # as podman names it: without it, podman logs in to its first search registry
|
|
29
|
+
_USER = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$")
|
|
30
|
+
_TOKEN_MAX = 4096
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def docker_state() -> dict:
|
|
34
|
+
"""{ok, version} when the engine answers - its name and version, ``podman
|
|
35
|
+
6.1.4`` - else {ok: False, error}."""
|
|
36
|
+
why = container.unusable()
|
|
37
|
+
if why:
|
|
38
|
+
return {"ok": False, "error": why}
|
|
39
|
+
r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
|
|
40
|
+
return {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _config_dir() -> str:
|
|
44
|
+
return os.environ.get("DOCKER_CONFIG") or os.path.join(os.path.expanduser("~"), ".docker")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def login_user(registry: str = REGISTRY) -> Optional[str]:
|
|
48
|
+
"""The username the engine holds a Docker Hub login for, or None. Read from
|
|
49
|
+
docker's config, or asked of the credential helper it names; the secret
|
|
50
|
+
that comes with it is dropped here, never kept or returned. Podman says the
|
|
51
|
+
username itself."""
|
|
52
|
+
if container.podman():
|
|
53
|
+
try:
|
|
54
|
+
r = container._docker(["login", "--get-login", HUB], timeout=30, check=False)
|
|
55
|
+
except container.Unavailable:
|
|
56
|
+
return None
|
|
57
|
+
return (r.stdout.strip() or None) if r.returncode == 0 else None
|
|
58
|
+
try:
|
|
59
|
+
with open(os.path.join(_config_dir(), "config.json")) as f:
|
|
60
|
+
config = json.load(f)
|
|
61
|
+
except (OSError, ValueError):
|
|
62
|
+
return None
|
|
63
|
+
auth = (config.get("auths") or {}).get(registry) or {}
|
|
64
|
+
if auth.get("auth"):
|
|
65
|
+
try:
|
|
66
|
+
return base64.b64decode(auth["auth"]).decode().split(":", 1)[0] or None
|
|
67
|
+
except (ValueError, UnicodeDecodeError):
|
|
68
|
+
return None
|
|
69
|
+
helper = (config.get("credHelpers") or {}).get(registry) or config.get("credsStore")
|
|
70
|
+
if not helper or not re.match(r"^[A-Za-z0-9_.-]+$", helper):
|
|
71
|
+
return None
|
|
72
|
+
try:
|
|
73
|
+
r = subprocess.run(["docker-credential-" + helper, "get"], input=registry, text=True,
|
|
74
|
+
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=15)
|
|
75
|
+
return (json.loads(r.stdout).get("Username") or None) if r.returncode == 0 else None
|
|
76
|
+
except (OSError, ValueError, subprocess.TimeoutExpired):
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def login(username: str, token: str) -> dict:
|
|
81
|
+
"""``docker login`` (``podman login ... docker.io``) to Docker Hub, the token over stdin: {ok, user} or
|
|
82
|
+
{ok: False, error}. Neither value is stored or logged here."""
|
|
83
|
+
username = (username or "").strip().lower()
|
|
84
|
+
if not _USER.match(username):
|
|
85
|
+
return {"ok": False, "error": "Enter your Docker Hub username."}
|
|
86
|
+
if not token or len(token) > _TOKEN_MAX or "\n" in token.strip():
|
|
87
|
+
return {"ok": False, "error": "Enter an access token."}
|
|
88
|
+
try:
|
|
89
|
+
r = container._docker(["login", "--username", username, "--password-stdin", *_hub()], input=token.strip() + "\n",
|
|
90
|
+
timeout=60, check=False)
|
|
91
|
+
except container.Unavailable as e:
|
|
92
|
+
return {"ok": False, "error": str(e)}
|
|
93
|
+
if r.returncode != 0:
|
|
94
|
+
said = [ln for ln in (r.stderr or r.stdout).strip().splitlines() if ln.strip()]
|
|
95
|
+
return {"ok": False, "error": (said[-1] if said else f"{container.DOCKER} login: exit {r.returncode}")[:300]}
|
|
96
|
+
return {"ok": True, "user": login_user() or username}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _hub() -> List[str]:
|
|
100
|
+
return [HUB] if container.podman() else []
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def logout() -> dict:
|
|
104
|
+
try:
|
|
105
|
+
r = container._docker(["logout", *_hub()], timeout=30, check=False)
|
|
106
|
+
except container.Unavailable as e:
|
|
107
|
+
return {"ok": False, "error": str(e)}
|
|
108
|
+
return {"ok": r.returncode == 0} if r.returncode == 0 else {"ok": False, "error": (r.stderr or r.stdout).strip()[-300:]}
|
|
109
|
+
|
nullius/secrets.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""A benchmark's licences: the secrets its environments need, kept per
|
|
2
|
+
benchmark outside the repository.
|
|
3
|
+
|
|
4
|
+
A licence belongs to the software of one benchmark's dataset, so it is kept
|
|
5
|
+
under that benchmark: ``secrets/<benchmark>/<NAME>`` in the user configuration directory
|
|
6
|
+
(``SECRETS_DIR`` moves the root). Two benchmarks may each ask for a
|
|
7
|
+
``LICENSE_KEY``; they are two files in two folders and never mix. Each
|
|
8
|
+
folder is 0700 and each entry a 0600 file holding the bytes it was given -
|
|
9
|
+
a licence file keeps its bytes, a key or a server address its text - so
|
|
10
|
+
nothing of it is ever in the settings file or in this process's environment.
|
|
11
|
+
A data provider's token (Settings → Data providers) has a folder of its own
|
|
12
|
+
(``PROVIDERS``): a credential declared with that ``provider`` is read from
|
|
13
|
+
there alone, never from the benchmark's (:func:`store_of`).
|
|
14
|
+
|
|
15
|
+
``index.json`` beside them says, per name, when it was set, when it expires
|
|
16
|
+
(when said), its kind and the file it came from - never the value. A page
|
|
17
|
+
reads :func:`status` or :func:`state`; only a run reads :func:`value` or
|
|
18
|
+
:func:`path`, when a unit starts (``standard/run.py``), or to send it to a
|
|
19
|
+
host over stdin (``execution``). Standard library only.
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import base64
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import tempfile
|
|
28
|
+
import time
|
|
29
|
+
from typing import Dict, Optional, Tuple, Union
|
|
30
|
+
from nullius import paths
|
|
31
|
+
|
|
32
|
+
KINDS = ("value", "file", "server")
|
|
33
|
+
INDEX = "index.json"
|
|
34
|
+
# Shared data-provider credentials remain in the owner-only secret store.
|
|
35
|
+
HUGGINGFACE = "data-provider-huggingface"
|
|
36
|
+
# The data providers a benchmark's credential may name (``provider`` in its ``needs_secrets``):
|
|
37
|
+
# the folder each one's token is kept in, and its name there.
|
|
38
|
+
PROVIDERS = {"huggingface": (HUGGINGFACE, "HF_TOKEN")}
|
|
39
|
+
_BENCH = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,79}$")
|
|
40
|
+
_NAME = re.compile(r"^[A-Z_][A-Z0-9_]{0,63}$")
|
|
41
|
+
_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
42
|
+
# A licence server: a host name or an IPv4 address, and its port.
|
|
43
|
+
_SERVER = re.compile(r"^(?=.{1,253}:)[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?:([0-9]{1,5})$")
|
|
44
|
+
MAX_BYTES = 1_000_000 # a licence file is small; anything larger is not one
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def root(env=os.environ) -> str:
|
|
48
|
+
return paths.secrets_dir(env)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _folder(bench: str, make: bool = False) -> str:
|
|
52
|
+
if not isinstance(bench, str) or not _BENCH.match(bench):
|
|
53
|
+
raise ValueError(f"{bench!r} is not a benchmark's key")
|
|
54
|
+
d = os.path.join(root(), bench)
|
|
55
|
+
if make:
|
|
56
|
+
os.makedirs(root(), mode=0o700, exist_ok=True)
|
|
57
|
+
os.makedirs(d, mode=0o700, exist_ok=True)
|
|
58
|
+
os.chmod(root(), 0o700)
|
|
59
|
+
os.chmod(d, 0o700)
|
|
60
|
+
return d
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _name(name: str) -> str:
|
|
64
|
+
if not isinstance(name, str) or not _NAME.match(name):
|
|
65
|
+
raise ValueError(f"{name!r} is not a licence's name (capitals, digits, underscores)")
|
|
66
|
+
return name
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _write(path: str, data: bytes) -> None:
|
|
70
|
+
"""Written whole under a temporary name beside it, 0600 from its first
|
|
71
|
+
byte, then renamed over the old one."""
|
|
72
|
+
fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=os.path.dirname(path))
|
|
73
|
+
try:
|
|
74
|
+
os.fchmod(fd, 0o600)
|
|
75
|
+
with os.fdopen(fd, "wb") as f:
|
|
76
|
+
f.write(data)
|
|
77
|
+
os.replace(tmp, path)
|
|
78
|
+
except BaseException:
|
|
79
|
+
try:
|
|
80
|
+
os.unlink(tmp)
|
|
81
|
+
except OSError:
|
|
82
|
+
pass
|
|
83
|
+
raise
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _index(bench: str) -> Dict[str, dict]:
|
|
87
|
+
try:
|
|
88
|
+
with open(os.path.join(_folder(bench), INDEX)) as f:
|
|
89
|
+
got = json.load(f)
|
|
90
|
+
return got if isinstance(got, dict) else {}
|
|
91
|
+
except (OSError, ValueError):
|
|
92
|
+
return {}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def problem(kind: str, value: Union[str, bytes]) -> Optional[str]:
|
|
96
|
+
"""Why ``value`` cannot be a licence of ``kind``, or None. A key is one
|
|
97
|
+
line; a server is ``host:port``; a file is at most a megabyte."""
|
|
98
|
+
if kind not in KINDS:
|
|
99
|
+
return f"a licence is one of {KINDS}"
|
|
100
|
+
data = value.encode() if isinstance(value, str) else bytes(value)
|
|
101
|
+
if not data:
|
|
102
|
+
return "it is empty"
|
|
103
|
+
if len(data) > MAX_BYTES:
|
|
104
|
+
return f"it is larger than {MAX_BYTES // 1000} kB"
|
|
105
|
+
if kind == "file":
|
|
106
|
+
return None
|
|
107
|
+
try:
|
|
108
|
+
text = data.decode()
|
|
109
|
+
except UnicodeDecodeError:
|
|
110
|
+
return "it is not text"
|
|
111
|
+
if "\n" in text or "\r" in text or "\0" in text:
|
|
112
|
+
return "it is one line"
|
|
113
|
+
if kind == "server":
|
|
114
|
+
m = _SERVER.match(text)
|
|
115
|
+
if not m or not 0 < int(m.group(1)) < 65536:
|
|
116
|
+
return "a licence server is host:port (a name or an IPv4 address, and its port)"
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def set(bench: str, name: str, value: Union[str, bytes], *, kind: str = "value", filename: str = "",
|
|
121
|
+
expires: str = "") -> None:
|
|
122
|
+
"""Keep ``value`` as ``bench``'s licence ``name``, replacing any before.
|
|
123
|
+
``expires``: YYYY-MM-DD, or empty. Refused (ValueError) when it is not a
|
|
124
|
+
licence of its kind."""
|
|
125
|
+
_name(name)
|
|
126
|
+
if isinstance(value, str):
|
|
127
|
+
value = value.strip()
|
|
128
|
+
why = problem(kind, value)
|
|
129
|
+
if why:
|
|
130
|
+
raise ValueError(f"{name}: {why}")
|
|
131
|
+
if expires and not _DATE.match(expires):
|
|
132
|
+
raise ValueError(f"{name}: an expiry date is YYYY-MM-DD")
|
|
133
|
+
d = _folder(bench, make=True)
|
|
134
|
+
_write(os.path.join(d, name), value.encode() if isinstance(value, str) else bytes(value))
|
|
135
|
+
idx = _index(bench)
|
|
136
|
+
idx[name] = {"on": time.strftime("%Y-%m-%d"), "kind": kind,
|
|
137
|
+
**({"expires": expires} if expires else {}),
|
|
138
|
+
**({"filename": os.path.basename(filename)[:120]} if filename else {})}
|
|
139
|
+
_write(os.path.join(d, INDEX), json.dumps(idx, indent=1, sort_keys=True).encode())
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def delete(bench: str, name: str) -> bool:
|
|
143
|
+
"""Remove ``bench``'s licence ``name``; whether there was one."""
|
|
144
|
+
_name(name)
|
|
145
|
+
d = _folder(bench)
|
|
146
|
+
try:
|
|
147
|
+
os.unlink(os.path.join(d, name))
|
|
148
|
+
there = True
|
|
149
|
+
except FileNotFoundError:
|
|
150
|
+
there = False
|
|
151
|
+
idx = _index(bench)
|
|
152
|
+
if idx.pop(name, None) is not None:
|
|
153
|
+
_write(os.path.join(d, INDEX), json.dumps(idx, indent=1, sort_keys=True).encode())
|
|
154
|
+
return there
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def status(bench: str) -> Dict[str, dict]:
|
|
158
|
+
"""Every licence ``bench`` holds: {name: {set, on, expires, kind,
|
|
159
|
+
filename, expired}} - dates and names, never a value."""
|
|
160
|
+
out = {}
|
|
161
|
+
d = _folder(bench)
|
|
162
|
+
today = time.strftime("%Y-%m-%d")
|
|
163
|
+
for name, rec in _index(bench).items():
|
|
164
|
+
if not _NAME.match(name) or not os.path.isfile(os.path.join(d, name)):
|
|
165
|
+
continue
|
|
166
|
+
exp = rec.get("expires") or ""
|
|
167
|
+
out[name] = {"set": True, "on": rec.get("on") or "", "expires": exp, "kind": rec.get("kind") or "value",
|
|
168
|
+
"filename": rec.get("filename") or "", "expired": bool(exp) and exp < today}
|
|
169
|
+
return out
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def path(bench: str, name: str) -> Optional[str]:
|
|
173
|
+
"""Where ``bench``'s licence ``name`` is on this machine, or None: what a
|
|
174
|
+
run mounts, or names to a process, for a licence file."""
|
|
175
|
+
p = os.path.join(_folder(bench), _name(name))
|
|
176
|
+
return p if os.path.isfile(p) else None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def value(bench: str, name: str) -> Optional[bytes]:
|
|
180
|
+
"""The bytes of ``bench``'s licence ``name``, or None. A run's alone."""
|
|
181
|
+
p = path(bench, name)
|
|
182
|
+
if p is None:
|
|
183
|
+
return None
|
|
184
|
+
with open(p, "rb") as f:
|
|
185
|
+
return f.read()
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def store_of(bench: str, name: str, provider: str = "") -> Tuple[str, str]:
|
|
189
|
+
"""Where ``bench``'s credential ``name`` is kept, (folder, name): its
|
|
190
|
+
data provider's alone when ``provider`` is said, else the benchmark's.
|
|
191
|
+
ValueError for a provider that is not one."""
|
|
192
|
+
if not provider:
|
|
193
|
+
return bench, name
|
|
194
|
+
if provider not in PROVIDERS:
|
|
195
|
+
raise ValueError(f"{provider!r} is not a data provider ({', '.join(PROVIDERS)})")
|
|
196
|
+
return PROVIDERS[provider]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def state(bench: str, name: str, provider: str = "") -> Optional[dict]:
|
|
200
|
+
""":func:`status` of one credential, read where it is kept
|
|
201
|
+
(:func:`store_of`); None when it is not set."""
|
|
202
|
+
folder, kept = store_of(bench, name, provider)
|
|
203
|
+
return status(folder).get(kept)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# What a run on a host is handed of the licences, one line of its launcher's stdin (``execution``).
|
|
207
|
+
HANDED = "NULLIUS_LICENCES"
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def unpack(line: str) -> Dict[str, bytes]:
|
|
211
|
+
"""Decode a host's licence line: base64 of {name: base64 bytes}.
|
|
212
|
+
Return {name: bytes}, or nothing for a line that is not one."""
|
|
213
|
+
try:
|
|
214
|
+
got = json.loads(base64.b64decode(line.encode(), validate=True))
|
|
215
|
+
return {n: base64.b64decode(v.encode(), validate=True) for n, v in got.items()
|
|
216
|
+
if isinstance(n, str) and _NAME.match(n) and isinstance(v, str)}
|
|
217
|
+
except (ValueError, AttributeError, TypeError):
|
|
218
|
+
return {}
|