nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Conformance: run a port's kit against an agent or a capability, and write down
|
|
2
|
+
what passed.
|
|
3
|
+
|
|
4
|
+
A port is offered only once its kit has passed (``definitions.problems``
|
|
5
|
+
refuses an ``offers`` entry without its ``proven`` record), so this is the one
|
|
6
|
+
way into ``offers``. A kit passes when every one of its **basic** checks
|
|
7
|
+
passes. Its **skills** (``definitions.skills_of``: the harder uses of the same
|
|
8
|
+
contract a kit may name) are recorded one by one, shown or not, and never take
|
|
9
|
+
the port away - a benchmark that relies on one says so in its
|
|
10
|
+
``needs_skills``. A kit that names no skill is every check basic: it passes
|
|
11
|
+
when every check has reward 1, as before skills existed.
|
|
12
|
+
|
|
13
|
+
``repeat``: each check run that many times, for confidence; a check, basic or
|
|
14
|
+
a skill, passes when it passes in a majority of its runs, and the record
|
|
15
|
+
counts them (``2/3``). One run each by default.
|
|
16
|
+
|
|
17
|
+
The record keeps the date, the score, the model it passed on, the version of
|
|
18
|
+
the agent, the version of the kit (``kit_version``: a kit grows stronger under
|
|
19
|
+
the same port, and a proof says which one it passed), the skills it showed or
|
|
20
|
+
missed, and the job that shows it - a claim anyone can open and read.
|
|
21
|
+
|
|
22
|
+
A port whose kit fails is taken out of ``offers`` when the definition is
|
|
23
|
+
written back: a claim that stopped being true does not stay.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import time
|
|
28
|
+
from typing import Callable, Dict, List, Optional
|
|
29
|
+
|
|
30
|
+
from nullius.runs import parse
|
|
31
|
+
from nullius.standard import definitions, ports, run
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
STAMP = "%Y-%m-%d__%H-%M-%S"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def job_name(stamp: str, kit: str, key: str) -> str:
|
|
38
|
+
return f"{stamp}__kit__{kit}__{key}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def candidates(other: dict) -> List[str]:
|
|
42
|
+
"""The ports of the other side's family that have a kit; a command agent
|
|
43
|
+
can only ever offer the ports its form allows."""
|
|
44
|
+
fam = other["_kind"]
|
|
45
|
+
out = [p for p, s in ports.PORTS.items() if ports.family(p) == fam and s["kit"]]
|
|
46
|
+
if fam == "capability":
|
|
47
|
+
out = [p for p in out if p.split(".", 1)[1].split("/")[0] == other.get("kind")]
|
|
48
|
+
if fam == "agent" and "command" in (other.get("start") or {}):
|
|
49
|
+
out = [p for p in out if p in ("agent.task/v1", "agent.workspace/v1", "agent.search/v1")]
|
|
50
|
+
return out
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def majority(passes: int, runs: int) -> bool:
|
|
54
|
+
"""Whether a check run ``runs`` times passed: in more than half of them."""
|
|
55
|
+
return runs > 0 and 2 * passes > runs
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def outcome(kit: dict, runs: Dict[str, List[bool]], repeat: int = 1) -> dict:
|
|
59
|
+
"""What a kit's runs prove, from each check's passes ({check: [passed,
|
|
60
|
+
...]}, one per run): ``passed``, the basic checks held of all of them;
|
|
61
|
+
``ok`` when every one held; ``skills``, each skill's count and whether it
|
|
62
|
+
held - only for a kit that names skills. With ``repeat`` over one,
|
|
63
|
+
``checks`` counts each basic check's runs too. A basic check the kit
|
|
64
|
+
names and the job never ran is a check not held."""
|
|
65
|
+
skills = definitions.skills_of(kit)
|
|
66
|
+
named = [u for u in kit.get("basic") or [] if u not in skills]
|
|
67
|
+
basic = named + [u for u in runs if u not in skills and u not in named]
|
|
68
|
+
|
|
69
|
+
def count(u):
|
|
70
|
+
got = runs.get(u) or []
|
|
71
|
+
return sum(1 for x in got if x), len(got)
|
|
72
|
+
|
|
73
|
+
held = [u for u in basic if majority(*count(u))]
|
|
74
|
+
out = {"passed": f"{len(held)}/{len(basic)}", "ok": bool(basic) and len(held) == len(basic)}
|
|
75
|
+
if repeat > 1:
|
|
76
|
+
out["repeat"] = repeat
|
|
77
|
+
out["checks"] = {u: "{}/{}".format(*count(u)) for u in basic}
|
|
78
|
+
if skills:
|
|
79
|
+
out["skills"] = {s: {"passed": "{}/{}".format(*count(s)), "ok": majority(*count(s))} for s in skills}
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _runs(job) -> Dict[str, List[bool]]:
|
|
84
|
+
"""Each check's passes in a kit's job, by the unit its trial ran (its task
|
|
85
|
+
name after the kit's key); a trial that ended in an error did not pass."""
|
|
86
|
+
out: Dict[str, List[bool]] = {}
|
|
87
|
+
for t in job.trials:
|
|
88
|
+
out.setdefault(t.task_name.split("/", 1)[-1], []).append(t.reward == 1.0)
|
|
89
|
+
return out
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def said(port: str, rec: dict) -> List[str]:
|
|
93
|
+
"""A record as the lines a shell shows: the port, then each skill."""
|
|
94
|
+
lines = [f"{port}: {'passed' if rec.get('ok', True) else 'FAILED'} {rec.get('passed', '')}"
|
|
95
|
+
+ (f" basic checks, each held in most of {rec['repeat']} runs" if rec.get("repeat") else "")]
|
|
96
|
+
lines += [f" {u}: {n}" for u, n in (rec.get("checks") or {}).items()]
|
|
97
|
+
skills = definitions.port_skills(port)
|
|
98
|
+
lines += [f" skill {s}: {'shown' if r.get('ok') else 'not shown'} {r.get('passed', '')}"
|
|
99
|
+
+ (f" - {skills[s]}" if s in skills else "") for s, r in (rec.get("skills") or {}).items()]
|
|
100
|
+
return lines
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] = None, jobs_root: str = "jobs",
|
|
104
|
+
write: bool = True, say: Callable[[str], None] = print, env=None, unsandboxed: bool = False,
|
|
105
|
+
stamp: Optional[str] = None, backend: str = "local", options: Optional[dict] = None,
|
|
106
|
+
repeat: int = 1) -> Dict[str, dict]:
|
|
107
|
+
"""Run the kits of ``which`` (every candidate port by default) and, with
|
|
108
|
+
``write``, record the result in the definition. Returns {port: record}.
|
|
109
|
+
Each kit's job is named ``<stamp>__kit__<kit>__<key>`` (:func:`job_name`),
|
|
110
|
+
``stamp`` being now unless given: a page that starts the check names the
|
|
111
|
+
jobs before they exist. ``backend``: where the kits run (``run.Run``).
|
|
112
|
+
``options``: the agent's options for the kits'
|
|
113
|
+
runs; a record proven with any of them off its default says which.
|
|
114
|
+
``repeat``: how many times each check runs (:func:`outcome`)."""
|
|
115
|
+
import os
|
|
116
|
+
env = env if env is not None else os.environ
|
|
117
|
+
repeat = max(1, int(repeat or 1))
|
|
118
|
+
got = {}
|
|
119
|
+
for p in which or candidates(other):
|
|
120
|
+
kit = definitions.kit(p)
|
|
121
|
+
if kit is None:
|
|
122
|
+
say(f"{p}: no kit yet")
|
|
123
|
+
continue
|
|
124
|
+
say(f"{p}: {kit['summary']}")
|
|
125
|
+
job_dir = run.run(kit, other, model=model, provider=provider, jobs_root=jobs_root, concurrency=2,
|
|
126
|
+
name=job_name(stamp or time.strftime(STAMP), kit["key"], definitions.job_word(other)),
|
|
127
|
+
say=lambda s: say(" " + s), env=env, proving=True, unsandboxed=unsandboxed,
|
|
128
|
+
backend=backend, options=options, repeat=repeat)
|
|
129
|
+
o = outcome(kit, _runs(parse.parse_job(job_dir)), repeat)
|
|
130
|
+
got[p] = {"passed": o["passed"], "ok": o["ok"], "on": time.strftime("%Y-%m-%d"), "model": model, "provider": provider,
|
|
131
|
+
"version": str((other.get("upstream") or {}).get("version") or ""),
|
|
132
|
+
"kit_version": definitions.kit_version(kit), "backend": backend, "job": os.path.basename(job_dir),
|
|
133
|
+
**{k: o[k] for k in ("repeat", "checks", "skills") if k in o}}
|
|
134
|
+
changed = {k: v for k, v in (options or {}).items() if v != ((other.get("options") or {}).get(k) or {}).get("default")}
|
|
135
|
+
if changed:
|
|
136
|
+
got[p]["options"] = changed
|
|
137
|
+
for line in said(p, got[p]):
|
|
138
|
+
say(" " + line)
|
|
139
|
+
if write:
|
|
140
|
+
def change(cur): # on the definition as it is now: another proof or an edit since the start stays
|
|
141
|
+
offers = [p for p in cur.get("offers") or [] if not (p in got and not got[p]["ok"])]
|
|
142
|
+
offers += [p for p, r in got.items() if r["ok"] and p not in offers]
|
|
143
|
+
# a port's record goes with its offer; another proof (fidelity) is not this run's to take back
|
|
144
|
+
proven = {p: r for p, r in (cur.get("proven") or {}).items() if p in offers or not ports.known(p)}
|
|
145
|
+
proven.update({p: {k: v for k, v in r.items() if k != "ok"} for p, r in got.items() if r["ok"]})
|
|
146
|
+
cur["offers"] = [p for p in ports.PORTS if p in offers]
|
|
147
|
+
cur["proven"] = proven
|
|
148
|
+
definitions.save(other, change)
|
|
149
|
+
return got
|