nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
{% block title %}Workspaces — Nullius{% endblock %}
|
|
3
|
+
{% block crumb %}/ workspaces{% endblock %}
|
|
4
|
+
{% block body %}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
<div class="page-title"><h1>Workspaces</h1></div>
|
|
8
|
+
<p class="subtitle">A named set of run roots and what was read from them. <a href="{{ url_for('import_page', new=1) }}">New workspace →</a></p>
|
|
9
|
+
|
|
10
|
+
{% if workspaces %}
|
|
11
|
+
<table class="grid">
|
|
12
|
+
<thead>
|
|
13
|
+
<tr><th></th><th>Workspace</th><th>Name</th><th>Roots</th><th class="num" title="Parsed runs held in the cache">Cached runs</th><th>Last opened</th><th></th><th></th></tr>
|
|
14
|
+
</thead>
|
|
15
|
+
<tbody>
|
|
16
|
+
{% for ws in workspaces %}
|
|
17
|
+
{% set current = workspace and ws.name == workspace.name %}
|
|
18
|
+
<tr class="{{ 'ws-current' if current }}">
|
|
19
|
+
<td class="ws-mark">{% if current %}<span class="ok" title="open">●</span>{% endif %}</td>
|
|
20
|
+
<td>
|
|
21
|
+
{# every workspace renames from its own row, open or not; its directory follows the name #}
|
|
22
|
+
<form method="post" action="{{ url_for('workspace_rename') }}" class="ws-rename">
|
|
23
|
+
<input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="name" value="{{ ws.name }}">
|
|
24
|
+
<input type="text" name="label" value="{{ ws.label }}" aria-label="Rename the workspace {{ ws.label }}" spellcheck="false" autocomplete="off">
|
|
25
|
+
<button class="linkish" type="submit">rename</button>
|
|
26
|
+
</form>
|
|
27
|
+
</td>
|
|
28
|
+
<td class="mono faint">{{ ws.name }}</td>
|
|
29
|
+
<td class="ws-roots" title="{{ ws.roots|join(' ') }}">{% for r in ws.roots %}<span class="mono">{{ r }}</span>{% if not loop.last %}<br>{% endif %}{% endfor %}</td>
|
|
30
|
+
<td class="num">{{ ws.n_cached }}</td>
|
|
31
|
+
<td class="mono faint">{{ ws.opened[:16]|replace('T', ' ') }}</td>
|
|
32
|
+
<td class="arrow">
|
|
33
|
+
{% if current %}<a href="{{ url_for('index') }}">runs →</a>
|
|
34
|
+
{% else %}<form method="post" action="{{ url_for('workspace_open', name=ws.name) }}" class="header-form"><input type="hidden" name="csrf" value="{{ csrf_token() }}"><button class="btn sm" type="submit">Open</button></form>{% endif %}
|
|
35
|
+
</td>
|
|
36
|
+
<td class="ws-delete">
|
|
37
|
+
{# removes the workspace's cache, imports and export record; the job roots stay where they are #}
|
|
38
|
+
<form method="post" action="{{ url_for('workspace_delete', name=ws.name) }}" class="header-form" data-confirm="Remove the workspace {{ ws.label }}? Its cached parses, converted imports and export record go; the job directories it read stay.">
|
|
39
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
40
|
+
<button class="ws-remove" type="submit" title="Remove this workspace (its job roots are untouched)" aria-label="Remove workspace {{ ws.label }}">×</button>
|
|
41
|
+
</form>
|
|
42
|
+
</td>
|
|
43
|
+
</tr>
|
|
44
|
+
{% endfor %}
|
|
45
|
+
</tbody>
|
|
46
|
+
</table>
|
|
47
|
+
{% else %}
|
|
48
|
+
<div class="alert">No workspace yet.</div>
|
|
49
|
+
{% endif %}
|
|
50
|
+
<script>
|
|
51
|
+
document.querySelectorAll('form[data-confirm]').forEach(f => f.addEventListener('submit', e => { if (!confirm(f.dataset.confirm)) e.preventDefault(); }));
|
|
52
|
+
</script>
|
|
53
|
+
|
|
54
|
+
{% endblock %}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The Flask side, one module per capability; `core` holds the application object, `shared` what the pages share."""
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""The Agents, Capabilities and Kits catalogues: the standard's agents and capabilities as cards,
|
|
2
|
+
each opening a page of its own - what it has proven, what it can run, and a
|
|
3
|
+
check of its ports.
|
|
4
|
+
|
|
5
|
+
Each entry is its definition in the catalogue (``catalogue/agents/<key>/``,
|
|
6
|
+
``catalogue/capabilities/<key>/``, ``docs/standard.md``): how Nullius talks to it,
|
|
7
|
+
what it installs, the ports it offers with the record of the kit that proved
|
|
8
|
+
each - the skills of that kit it showed or not, and the job that shows it,
|
|
9
|
+
when the open workspace holds that job - and
|
|
10
|
+
every benchmark of its family, the ones it fits linked to a builder recipe for
|
|
11
|
+
the pair and the others with the reasons. *Prove its ports* runs its conformance
|
|
12
|
+
kits as a run of this machine (``standard/recipe.py``, ``conformance``), the
|
|
13
|
+
record rewritten by what passes.
|
|
14
|
+
|
|
15
|
+
An agent or a capability with several versions is one card, its default
|
|
16
|
+
version's; its page shows one version at a time (``?v=``, the default
|
|
17
|
+
without it), with the list of its versions and what each has proven.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import io
|
|
22
|
+
import keyword
|
|
23
|
+
import os
|
|
24
|
+
import tokenize
|
|
25
|
+
import time
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from flask import abort, flash, redirect, render_template, request, url_for
|
|
29
|
+
from nullius import execution, paths, providers
|
|
30
|
+
from nullius.catalogue import storage
|
|
31
|
+
from nullius.standard import conform, definitions, howto, match, ports
|
|
32
|
+
from nullius.standard import recipe as standard_recipe
|
|
33
|
+
|
|
34
|
+
from .core import app
|
|
35
|
+
from . import shared
|
|
36
|
+
from . import execute
|
|
37
|
+
from . import settings as settings_web
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _install_words(o: dict) -> list:
|
|
41
|
+
"""What a definition installs, in a few words each."""
|
|
42
|
+
inst, up = o.get("install") or {}, o.get("upstream") or {}
|
|
43
|
+
out = [f"npm {p}" for p in inst.get("npm") or []] + [f"pip {p}" for p in inst.get("pip") or []]
|
|
44
|
+
if inst.get("project"):
|
|
45
|
+
extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
|
|
46
|
+
out.append(f"{inst['project']}{extras} from {up.get('repository', '').removeprefix('https://')} at {(up['tag'] + ' (' + str(up.get('tested', ''))[:12] + ')') if up.get('tag') else str(up.get('tested', ''))[:12]}")
|
|
47
|
+
if inst.get("python"):
|
|
48
|
+
out.append(f"on Python {inst['python']}")
|
|
49
|
+
out += [f"{', '.join(x['packages'])} from {x['url'].removeprefix('https://')}" for x in inst.get("indexes") or []]
|
|
50
|
+
out += [f"the model {m}" for m in inst.get("prefetch") or []]
|
|
51
|
+
return out
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _entries(kind: str, every: bool = False) -> dict:
|
|
55
|
+
others, bad = definitions.load(kind, every=every)
|
|
56
|
+
benches = [b for b in definitions.load("benchmark")[0] if b["family"] == kind]
|
|
57
|
+
jobs = {j.job_id for j in shared._jobs_for_request()}
|
|
58
|
+
out = []
|
|
59
|
+
for o in others:
|
|
60
|
+
cand = set(conform.candidates(o))
|
|
61
|
+
rows = []
|
|
62
|
+
for p, spec in ports.PORTS.items():
|
|
63
|
+
if ports.family(p) != kind or (kind == "capability" and p.split(".", 1)[1].split("/")[0] != o.get("kind")):
|
|
64
|
+
continue
|
|
65
|
+
rec = (o.get("proven") or {}).get(p) if p in (o.get("offers") or []) else None
|
|
66
|
+
state = "proven" if rec else ("not offered" if p in cand else ("no kit yet" if not spec["kit"] else "not for it"))
|
|
67
|
+
kit = definitions.kit(p) if spec["kit"] else None
|
|
68
|
+
rows.append({"port": p, "label": spec.get("label") or p, "summary": spec["summary"], "state": state, "record": rec,
|
|
69
|
+
"proof_kit": definitions.proof_kit(rec) if rec else None,
|
|
70
|
+
"kit": {"says": kit["summary"], "checks": kit["datasets"][0]["units"], "key": kit["key"], "version": definitions.kit_version(kit)} if kit else None,
|
|
71
|
+
"job": rec["job"] if rec and rec.get("job") in jobs else None,
|
|
72
|
+
"older_kit": definitions.older_kit(p, rec),
|
|
73
|
+
"skills": standard_recipe.skills_of(p, rec)})
|
|
74
|
+
fits = [{"key": b["key"], "label": b["label"], "needs": b["needs"], "why": match.fit(b, o)} for b in benches]
|
|
75
|
+
fid = (o.get("proven") or {}).get("fidelity")
|
|
76
|
+
out.append({"d": o, "protocol": standard_recipe.protocol(o), "ports": rows, "fits": fits,
|
|
77
|
+
"install": _install_words(o),
|
|
78
|
+
# its options, each with its default and what else it takes; its fidelity proof, when it has one
|
|
79
|
+
"options": [{"name": n, "summary": x.get("summary", ""), "default": x.get("default"),
|
|
80
|
+
"values": list(x.get("values") or []), "pattern": x.get("pattern", "")}
|
|
81
|
+
for n, x in (o.get("options") or {}).items()],
|
|
82
|
+
"fidelity": fid if isinstance(fid, dict) else None})
|
|
83
|
+
out.sort(key=lambda e: (not e["d"].get("example", False), definitions.title(e["d"]).casefold()))
|
|
84
|
+
return {"entries": out, "problems": bad}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _page(kind: str):
|
|
88
|
+
example = os.path.join(definitions.CATALOGUE, "agents", "reference") if kind == "agent" else \
|
|
89
|
+
os.path.join(definitions.CATALOGUE, "capabilities", "keyword-memory")
|
|
90
|
+
return render_template("agents/standard.html", kind=kind, how=howto.steps(kind), prompts=howto.prompts(),
|
|
91
|
+
catalogue_dir=paths.catalogue_dir(), example_dir=example, **_entries(kind))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _entry(kind: str, key: str):
|
|
95
|
+
every = _entries(kind, every=True)["entries"]
|
|
96
|
+
mine = [x for x in every if x["d"]["key"] == key]
|
|
97
|
+
v = request.args.get("v")
|
|
98
|
+
e = next((x for x in mine if x["d"]["_version"] == v), None) if v else \
|
|
99
|
+
next((x for x in mine if x["d"]["_version"] == x["d"]["_default"]), None)
|
|
100
|
+
if e is None:
|
|
101
|
+
abort(404)
|
|
102
|
+
versions = [{"v": x["d"]["_version"], "default": x["d"]["_version"] == x["d"]["_default"], "current": x is e,
|
|
103
|
+
"proven": sum(1 for r in x["ports"] if r["state"] == "proven"), "ports": len(x["ports"])} for x in mine]
|
|
104
|
+
have = providers.available(os.environ)
|
|
105
|
+
return render_template("agents/entry.html", kind=kind, e=e, versions=versions if len(mine) > 1 else [],
|
|
106
|
+
restore_bundle=storage.can_restore(definitions.KINDS[kind]["dir"], key, definitions.CATALOGUE),
|
|
107
|
+
title=definitions.title(e["d"]), providers=[(p, providers.label(p)) for p in have],
|
|
108
|
+
local=settings_web._local_client(), csrf=shared._csrf_token())
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@app.route("/agents")
|
|
112
|
+
def agents_page():
|
|
113
|
+
return _page("agent")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@app.route("/features")
|
|
117
|
+
@app.route("/capabilities")
|
|
118
|
+
def features_page():
|
|
119
|
+
return _page("capability")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@app.route("/agents/<key>")
|
|
123
|
+
def agent_page(key):
|
|
124
|
+
return _entry("agent", key)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@app.route("/features/<key>")
|
|
128
|
+
@app.route("/capabilities/<key>")
|
|
129
|
+
def feature_page(key):
|
|
130
|
+
return _entry("capability", key)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _check(kind: str, key: str):
|
|
134
|
+
"""Start the conformance kits of one agent or capability, here, as a run;
|
|
135
|
+
then its first kit's job page, which follows the run."""
|
|
136
|
+
shared._require_csrf()
|
|
137
|
+
if not settings_web._local_client():
|
|
138
|
+
abort(403, description="Checks are started from a browser on the application host only.")
|
|
139
|
+
version = request.form.get("version", "")
|
|
140
|
+
back = url_for("agent_page" if kind == "agent" else "feature_page", key=key, **({"v": version} if version else {}))
|
|
141
|
+
provider, endpoint = providers.split(request.form.get("provider", ""))
|
|
142
|
+
backend = definitions.names.backend(request.form.get("backend", "local"))
|
|
143
|
+
stamp = time.strftime(conform.STAMP)
|
|
144
|
+
got = standard_recipe.conformance(kind, f"{key}@{version}" if version else key, model=request.form.get("model", "").strip(), provider=provider,
|
|
145
|
+
unsandboxed=request.form.get("unsandboxed") == "yes", stamp=stamp, backend=backend)
|
|
146
|
+
problem = " ".join(got["errors"]) or execute.provider_problem(provider, endpoint)
|
|
147
|
+
if not problem and backend == "container":
|
|
148
|
+
problem = execute.docker_problem()
|
|
149
|
+
if problem:
|
|
150
|
+
flash(problem, "error")
|
|
151
|
+
return redirect(back)
|
|
152
|
+
word = f"{key}-{version}" if version else key
|
|
153
|
+
plan = execution.Plan(name=f"check-{word}-{time.strftime('%m%d-%H%M%S')}", script=got["script"], runner=standard_recipe.HARNESS, docker=backend == "container",
|
|
154
|
+
**execute.credentials_for(provider, endpoint), model=request.form.get("model", "").strip(), jobs=got["jobs"],
|
|
155
|
+
parts=[{"id": word, "label": got["label"] + "'s kits", "script": got["script"]}])
|
|
156
|
+
problem = plan.check() or (execution.name_taken(plan.name) if os.path.exists(os.path.join(execution.runs_root(), plan.name)) else None)
|
|
157
|
+
if problem:
|
|
158
|
+
flash(problem, "error")
|
|
159
|
+
return redirect(back)
|
|
160
|
+
ex = execution.start(plan)
|
|
161
|
+
shared.adopt_roots([ex.jobs])
|
|
162
|
+
n = len(got["jobs"])
|
|
163
|
+
flash(f"Checking {got['label']}'s ports: " + (f"{n} kits, one after the other." if n > 1 else "its one kit."), "ok")
|
|
164
|
+
return redirect(url_for("job", job_id=got["jobs"][0]["name"]))
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@app.route("/agents/<key>/check", methods=["POST"])
|
|
168
|
+
def agent_check(key):
|
|
169
|
+
return _check("agent", key)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@app.route("/features/<key>/check", methods=["POST"])
|
|
173
|
+
@app.route("/capabilities/<key>/check", methods=["POST"])
|
|
174
|
+
def feature_check(key):
|
|
175
|
+
return _check("capability", key)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _kit_entries():
|
|
179
|
+
shipped = {k["key"]: k for k in definitions.kits()}
|
|
180
|
+
return [dict(port=port, label=p["label"], d=shipped[p["kit"]],
|
|
181
|
+
version=definitions.kit_version(shipped[p["kit"]]))
|
|
182
|
+
for port, p in ports.PORTS.items() if p.get("kit") in shipped]
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@app.route("/kits")
|
|
186
|
+
def kits_page():
|
|
187
|
+
return render_template("agents/kits.html", entries=_kit_entries(), entry=None)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _python_tokens(code):
|
|
191
|
+
"""Colour source without executing it or introducing trusted HTML."""
|
|
192
|
+
offsets = [0]
|
|
193
|
+
for line in code.splitlines(keepends=True):
|
|
194
|
+
offsets.append(offsets[-1] + len(line))
|
|
195
|
+
chunks, cursor = [], 0
|
|
196
|
+
try:
|
|
197
|
+
tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))
|
|
198
|
+
except (tokenize.TokenError, IndentationError):
|
|
199
|
+
return [{"kind": "", "text": code}]
|
|
200
|
+
for token in tokens:
|
|
201
|
+
start = offsets[min(token.start[0] - 1, len(offsets) - 1)] + token.start[1]
|
|
202
|
+
end = offsets[min(token.end[0] - 1, len(offsets) - 1)] + token.end[1]
|
|
203
|
+
if end <= start or start < cursor:
|
|
204
|
+
continue
|
|
205
|
+
if start > cursor:
|
|
206
|
+
chunks.append({"kind": "", "text": code[cursor:start]})
|
|
207
|
+
kind = ("comment" if token.type == tokenize.COMMENT else
|
|
208
|
+
"string" if token.type == tokenize.STRING or tokenize.tok_name[token.type].startswith("FSTRING") else
|
|
209
|
+
"number" if token.type == tokenize.NUMBER else
|
|
210
|
+
"keyword" if token.type == tokenize.NAME and keyword.iskeyword(token.string) else
|
|
211
|
+
"operator" if token.type == tokenize.OP else "")
|
|
212
|
+
chunks.append({"kind": kind, "text": code[start:end]})
|
|
213
|
+
cursor = end
|
|
214
|
+
if cursor < len(code):
|
|
215
|
+
chunks.append({"kind": "", "text": code[cursor:]})
|
|
216
|
+
return chunks
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@app.route("/kits/<key>")
|
|
220
|
+
def kit_page(key):
|
|
221
|
+
entry = next((e for e in _kit_entries() if e["d"]["key"] == key), None)
|
|
222
|
+
if entry is None:
|
|
223
|
+
abort(404)
|
|
224
|
+
# Read only shipped source files after resolving a known kit; never run a driver to render a page.
|
|
225
|
+
folder = Path(definitions.KITS) / entry["d"]["key"]
|
|
226
|
+
entry["sources"] = [{"name": source.name, "code": source.read_text(encoding="utf-8")}
|
|
227
|
+
for source in sorted(folder.glob("*.py"))]
|
|
228
|
+
for source in entry["sources"]:
|
|
229
|
+
source["tokens"] = _python_tokens(source["code"])
|
|
230
|
+
return render_template("agents/kits.html", entry=entry, entries=[])
|