nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,54 @@
1
+ {% extends "base.html" %}
2
+ {% block title %}Workspaces — Nullius{% endblock %}
3
+ {% block crumb %}/ workspaces{% endblock %}
4
+ {% block body %}
5
+
6
+
7
+ <div class="page-title"><h1>Workspaces</h1></div>
8
+ <p class="subtitle">A named set of run roots and what was read from them. <a href="{{ url_for('import_page', new=1) }}">New workspace →</a></p>
9
+
10
+ {% if workspaces %}
11
+ <table class="grid">
12
+ <thead>
13
+ <tr><th></th><th>Workspace</th><th>Name</th><th>Roots</th><th class="num" title="Parsed runs held in the cache">Cached runs</th><th>Last opened</th><th></th><th></th></tr>
14
+ </thead>
15
+ <tbody>
16
+ {% for ws in workspaces %}
17
+ {% set current = workspace and ws.name == workspace.name %}
18
+ <tr class="{{ 'ws-current' if current }}">
19
+ <td class="ws-mark">{% if current %}<span class="ok" title="open">●</span>{% endif %}</td>
20
+ <td>
21
+ {# every workspace renames from its own row, open or not; its directory follows the name #}
22
+ <form method="post" action="{{ url_for('workspace_rename') }}" class="ws-rename">
23
+ <input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="name" value="{{ ws.name }}">
24
+ <input type="text" name="label" value="{{ ws.label }}" aria-label="Rename the workspace {{ ws.label }}" spellcheck="false" autocomplete="off">
25
+ <button class="linkish" type="submit">rename</button>
26
+ </form>
27
+ </td>
28
+ <td class="mono faint">{{ ws.name }}</td>
29
+ <td class="ws-roots" title="{{ ws.roots|join('&#10;') }}">{% for r in ws.roots %}<span class="mono">{{ r }}</span>{% if not loop.last %}<br>{% endif %}{% endfor %}</td>
30
+ <td class="num">{{ ws.n_cached }}</td>
31
+ <td class="mono faint">{{ ws.opened[:16]|replace('T', ' ') }}</td>
32
+ <td class="arrow">
33
+ {% if current %}<a href="{{ url_for('index') }}">runs →</a>
34
+ {% else %}<form method="post" action="{{ url_for('workspace_open', name=ws.name) }}" class="header-form"><input type="hidden" name="csrf" value="{{ csrf_token() }}"><button class="btn sm" type="submit">Open</button></form>{% endif %}
35
+ </td>
36
+ <td class="ws-delete">
37
+ {# removes the workspace's cache, imports and export record; the job roots stay where they are #}
38
+ <form method="post" action="{{ url_for('workspace_delete', name=ws.name) }}" class="header-form" data-confirm="Remove the workspace {{ ws.label }}? Its cached parses, converted imports and export record go; the job directories it read stay.">
39
+ <input type="hidden" name="csrf" value="{{ csrf }}">
40
+ <button class="ws-remove" type="submit" title="Remove this workspace (its job roots are untouched)" aria-label="Remove workspace {{ ws.label }}">×</button>
41
+ </form>
42
+ </td>
43
+ </tr>
44
+ {% endfor %}
45
+ </tbody>
46
+ </table>
47
+ {% else %}
48
+ <div class="alert">No workspace yet.</div>
49
+ {% endif %}
50
+ <script>
51
+ document.querySelectorAll('form[data-confirm]').forEach(f => f.addEventListener('submit', e => { if (!confirm(f.dataset.confirm)) e.preventDefault(); }));
52
+ </script>
53
+
54
+ {% endblock %}
@@ -0,0 +1 @@
1
+ """The Flask side, one module per capability; `core` holds the application object, `shared` what the pages share."""
@@ -0,0 +1,230 @@
1
+ """The Agents, Capabilities and Kits catalogues: the standard's agents and capabilities as cards,
2
+ each opening a page of its own - what it has proven, what it can run, and a
3
+ check of its ports.
4
+
5
+ Each entry is its definition in the catalogue (``catalogue/agents/<key>/``,
6
+ ``catalogue/capabilities/<key>/``, ``docs/standard.md``): how Nullius talks to it,
7
+ what it installs, the ports it offers with the record of the kit that proved
8
+ each - the skills of that kit it showed or not, and the job that shows it,
9
+ when the open workspace holds that job - and
10
+ every benchmark of its family, the ones it fits linked to a builder recipe for
11
+ the pair and the others with the reasons. *Prove its ports* runs its conformance
12
+ kits as a run of this machine (``standard/recipe.py``, ``conformance``), the
13
+ record rewritten by what passes.
14
+
15
+ An agent or a capability with several versions is one card, its default
16
+ version's; its page shows one version at a time (``?v=``, the default
17
+ without it), with the list of its versions and what each has proven.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import io
22
+ import keyword
23
+ import os
24
+ import tokenize
25
+ import time
26
+ from pathlib import Path
27
+
28
+ from flask import abort, flash, redirect, render_template, request, url_for
29
+ from nullius import execution, paths, providers
30
+ from nullius.catalogue import storage
31
+ from nullius.standard import conform, definitions, howto, match, ports
32
+ from nullius.standard import recipe as standard_recipe
33
+
34
+ from .core import app
35
+ from . import shared
36
+ from . import execute
37
+ from . import settings as settings_web
38
+
39
+
40
+ def _install_words(o: dict) -> list:
41
+ """What a definition installs, in a few words each."""
42
+ inst, up = o.get("install") or {}, o.get("upstream") or {}
43
+ out = [f"npm {p}" for p in inst.get("npm") or []] + [f"pip {p}" for p in inst.get("pip") or []]
44
+ if inst.get("project"):
45
+ extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
46
+ out.append(f"{inst['project']}{extras} from {up.get('repository', '').removeprefix('https://')} at {(up['tag'] + ' (' + str(up.get('tested', ''))[:12] + ')') if up.get('tag') else str(up.get('tested', ''))[:12]}")
47
+ if inst.get("python"):
48
+ out.append(f"on Python {inst['python']}")
49
+ out += [f"{', '.join(x['packages'])} from {x['url'].removeprefix('https://')}" for x in inst.get("indexes") or []]
50
+ out += [f"the model {m}" for m in inst.get("prefetch") or []]
51
+ return out
52
+
53
+
54
+ def _entries(kind: str, every: bool = False) -> dict:
55
+ others, bad = definitions.load(kind, every=every)
56
+ benches = [b for b in definitions.load("benchmark")[0] if b["family"] == kind]
57
+ jobs = {j.job_id for j in shared._jobs_for_request()}
58
+ out = []
59
+ for o in others:
60
+ cand = set(conform.candidates(o))
61
+ rows = []
62
+ for p, spec in ports.PORTS.items():
63
+ if ports.family(p) != kind or (kind == "capability" and p.split(".", 1)[1].split("/")[0] != o.get("kind")):
64
+ continue
65
+ rec = (o.get("proven") or {}).get(p) if p in (o.get("offers") or []) else None
66
+ state = "proven" if rec else ("not offered" if p in cand else ("no kit yet" if not spec["kit"] else "not for it"))
67
+ kit = definitions.kit(p) if spec["kit"] else None
68
+ rows.append({"port": p, "label": spec.get("label") or p, "summary": spec["summary"], "state": state, "record": rec,
69
+ "proof_kit": definitions.proof_kit(rec) if rec else None,
70
+ "kit": {"says": kit["summary"], "checks": kit["datasets"][0]["units"], "key": kit["key"], "version": definitions.kit_version(kit)} if kit else None,
71
+ "job": rec["job"] if rec and rec.get("job") in jobs else None,
72
+ "older_kit": definitions.older_kit(p, rec),
73
+ "skills": standard_recipe.skills_of(p, rec)})
74
+ fits = [{"key": b["key"], "label": b["label"], "needs": b["needs"], "why": match.fit(b, o)} for b in benches]
75
+ fid = (o.get("proven") or {}).get("fidelity")
76
+ out.append({"d": o, "protocol": standard_recipe.protocol(o), "ports": rows, "fits": fits,
77
+ "install": _install_words(o),
78
+ # its options, each with its default and what else it takes; its fidelity proof, when it has one
79
+ "options": [{"name": n, "summary": x.get("summary", ""), "default": x.get("default"),
80
+ "values": list(x.get("values") or []), "pattern": x.get("pattern", "")}
81
+ for n, x in (o.get("options") or {}).items()],
82
+ "fidelity": fid if isinstance(fid, dict) else None})
83
+ out.sort(key=lambda e: (not e["d"].get("example", False), definitions.title(e["d"]).casefold()))
84
+ return {"entries": out, "problems": bad}
85
+
86
+
87
+ def _page(kind: str):
88
+ example = os.path.join(definitions.CATALOGUE, "agents", "reference") if kind == "agent" else \
89
+ os.path.join(definitions.CATALOGUE, "capabilities", "keyword-memory")
90
+ return render_template("agents/standard.html", kind=kind, how=howto.steps(kind), prompts=howto.prompts(),
91
+ catalogue_dir=paths.catalogue_dir(), example_dir=example, **_entries(kind))
92
+
93
+
94
+ def _entry(kind: str, key: str):
95
+ every = _entries(kind, every=True)["entries"]
96
+ mine = [x for x in every if x["d"]["key"] == key]
97
+ v = request.args.get("v")
98
+ e = next((x for x in mine if x["d"]["_version"] == v), None) if v else \
99
+ next((x for x in mine if x["d"]["_version"] == x["d"]["_default"]), None)
100
+ if e is None:
101
+ abort(404)
102
+ versions = [{"v": x["d"]["_version"], "default": x["d"]["_version"] == x["d"]["_default"], "current": x is e,
103
+ "proven": sum(1 for r in x["ports"] if r["state"] == "proven"), "ports": len(x["ports"])} for x in mine]
104
+ have = providers.available(os.environ)
105
+ return render_template("agents/entry.html", kind=kind, e=e, versions=versions if len(mine) > 1 else [],
106
+ restore_bundle=storage.can_restore(definitions.KINDS[kind]["dir"], key, definitions.CATALOGUE),
107
+ title=definitions.title(e["d"]), providers=[(p, providers.label(p)) for p in have],
108
+ local=settings_web._local_client(), csrf=shared._csrf_token())
109
+
110
+
111
+ @app.route("/agents")
112
+ def agents_page():
113
+ return _page("agent")
114
+
115
+
116
+ @app.route("/features")
117
+ @app.route("/capabilities")
118
+ def features_page():
119
+ return _page("capability")
120
+
121
+
122
+ @app.route("/agents/<key>")
123
+ def agent_page(key):
124
+ return _entry("agent", key)
125
+
126
+
127
+ @app.route("/features/<key>")
128
+ @app.route("/capabilities/<key>")
129
+ def feature_page(key):
130
+ return _entry("capability", key)
131
+
132
+
133
+ def _check(kind: str, key: str):
134
+ """Start the conformance kits of one agent or capability, here, as a run;
135
+ then its first kit's job page, which follows the run."""
136
+ shared._require_csrf()
137
+ if not settings_web._local_client():
138
+ abort(403, description="Checks are started from a browser on the application host only.")
139
+ version = request.form.get("version", "")
140
+ back = url_for("agent_page" if kind == "agent" else "feature_page", key=key, **({"v": version} if version else {}))
141
+ provider, endpoint = providers.split(request.form.get("provider", ""))
142
+ backend = definitions.names.backend(request.form.get("backend", "local"))
143
+ stamp = time.strftime(conform.STAMP)
144
+ got = standard_recipe.conformance(kind, f"{key}@{version}" if version else key, model=request.form.get("model", "").strip(), provider=provider,
145
+ unsandboxed=request.form.get("unsandboxed") == "yes", stamp=stamp, backend=backend)
146
+ problem = " ".join(got["errors"]) or execute.provider_problem(provider, endpoint)
147
+ if not problem and backend == "container":
148
+ problem = execute.docker_problem()
149
+ if problem:
150
+ flash(problem, "error")
151
+ return redirect(back)
152
+ word = f"{key}-{version}" if version else key
153
+ plan = execution.Plan(name=f"check-{word}-{time.strftime('%m%d-%H%M%S')}", script=got["script"], runner=standard_recipe.HARNESS, docker=backend == "container",
154
+ **execute.credentials_for(provider, endpoint), model=request.form.get("model", "").strip(), jobs=got["jobs"],
155
+ parts=[{"id": word, "label": got["label"] + "'s kits", "script": got["script"]}])
156
+ problem = plan.check() or (execution.name_taken(plan.name) if os.path.exists(os.path.join(execution.runs_root(), plan.name)) else None)
157
+ if problem:
158
+ flash(problem, "error")
159
+ return redirect(back)
160
+ ex = execution.start(plan)
161
+ shared.adopt_roots([ex.jobs])
162
+ n = len(got["jobs"])
163
+ flash(f"Checking {got['label']}'s ports: " + (f"{n} kits, one after the other." if n > 1 else "its one kit."), "ok")
164
+ return redirect(url_for("job", job_id=got["jobs"][0]["name"]))
165
+
166
+
167
+ @app.route("/agents/<key>/check", methods=["POST"])
168
+ def agent_check(key):
169
+ return _check("agent", key)
170
+
171
+
172
+ @app.route("/features/<key>/check", methods=["POST"])
173
+ @app.route("/capabilities/<key>/check", methods=["POST"])
174
+ def feature_check(key):
175
+ return _check("capability", key)
176
+
177
+
178
+ def _kit_entries():
179
+ shipped = {k["key"]: k for k in definitions.kits()}
180
+ return [dict(port=port, label=p["label"], d=shipped[p["kit"]],
181
+ version=definitions.kit_version(shipped[p["kit"]]))
182
+ for port, p in ports.PORTS.items() if p.get("kit") in shipped]
183
+
184
+
185
+ @app.route("/kits")
186
+ def kits_page():
187
+ return render_template("agents/kits.html", entries=_kit_entries(), entry=None)
188
+
189
+
190
+ def _python_tokens(code):
191
+ """Colour source without executing it or introducing trusted HTML."""
192
+ offsets = [0]
193
+ for line in code.splitlines(keepends=True):
194
+ offsets.append(offsets[-1] + len(line))
195
+ chunks, cursor = [], 0
196
+ try:
197
+ tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))
198
+ except (tokenize.TokenError, IndentationError):
199
+ return [{"kind": "", "text": code}]
200
+ for token in tokens:
201
+ start = offsets[min(token.start[0] - 1, len(offsets) - 1)] + token.start[1]
202
+ end = offsets[min(token.end[0] - 1, len(offsets) - 1)] + token.end[1]
203
+ if end <= start or start < cursor:
204
+ continue
205
+ if start > cursor:
206
+ chunks.append({"kind": "", "text": code[cursor:start]})
207
+ kind = ("comment" if token.type == tokenize.COMMENT else
208
+ "string" if token.type == tokenize.STRING or tokenize.tok_name[token.type].startswith("FSTRING") else
209
+ "number" if token.type == tokenize.NUMBER else
210
+ "keyword" if token.type == tokenize.NAME and keyword.iskeyword(token.string) else
211
+ "operator" if token.type == tokenize.OP else "")
212
+ chunks.append({"kind": kind, "text": code[start:end]})
213
+ cursor = end
214
+ if cursor < len(code):
215
+ chunks.append({"kind": "", "text": code[cursor:]})
216
+ return chunks
217
+
218
+
219
+ @app.route("/kits/<key>")
220
+ def kit_page(key):
221
+ entry = next((e for e in _kit_entries() if e["d"]["key"] == key), None)
222
+ if entry is None:
223
+ abort(404)
224
+ # Read only shipped source files after resolving a known kit; never run a driver to render a page.
225
+ folder = Path(definitions.KITS) / entry["d"]["key"]
226
+ entry["sources"] = [{"name": source.name, "code": source.read_text(encoding="utf-8")}
227
+ for source in sorted(folder.glob("*.py"))]
228
+ for source in entry["sources"]:
229
+ source["tokens"] = _python_tokens(source["code"])
230
+ return render_template("agents/kits.html", entry=entry, entries=[])