nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/workspace.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""Workspaces: a named set of roots, and what the app made from them.
|
|
2
|
+
|
|
3
|
+
A workspace gives the app's roots, and what it made from them, a name and a
|
|
4
|
+
directory, so they survive a restart, so two sets of runs can be kept apart,
|
|
5
|
+
and so the expensive part — parsing a 375-trial job, converting a foreign
|
|
6
|
+
run tree — happens once per change rather than once per page.
|
|
7
|
+
|
|
8
|
+
<home>/<name>/workspace.json name, label, roots, created, opened
|
|
9
|
+
<home>/<name>/imported/ foreign run trees converted to job dirs
|
|
10
|
+
<home>/<name>/jobs/<id>.json a parsed Job, keyed by the job dir's signature
|
|
11
|
+
|
|
12
|
+
``home`` is ``WORKSPACES_DIR`` or ``workspaces/`` in the persistent user data directory.
|
|
13
|
+
A name is a slug the user typed, or two words picked at random — something
|
|
14
|
+
to say out loud, the way Harbor suffixes a trial with ``__TKLnaSW`` and
|
|
15
|
+
nobody can.
|
|
16
|
+
|
|
17
|
+
The cache is keyed by a *signature* of the job directory (every result.json
|
|
18
|
+
and trajectory.json under it, with size and mtime), never by time: a run
|
|
19
|
+
still going changes its files, the signature moves, the job is parsed again.
|
|
20
|
+
A cached job that was parsed by an older version of ``parse`` is refreshed
|
|
21
|
+
too — the signature carries ``parse.SCHEMA`` so a change there invalidates.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import shutil
|
|
29
|
+
import random
|
|
30
|
+
import re
|
|
31
|
+
import threading
|
|
32
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
33
|
+
from datetime import datetime, timezone
|
|
34
|
+
|
|
35
|
+
from nullius.runs import parse
|
|
36
|
+
from nullius import paths
|
|
37
|
+
|
|
38
|
+
ADJECTIVES = """brisk quiet tidy sturdy curious gentle nimble patient plucky sunny wry keen
|
|
39
|
+
mellow bold calm clever crisp eager fuzzy glossy humble jolly lucky merry neat proud quirky
|
|
40
|
+
rapid sharp snug spry steady swift warm witty zesty cosmic dapper mossy""".split()
|
|
41
|
+
NOUNS = """otter quasar kernel lantern marmot nebula pixel socket walrus badger comet dynamo
|
|
42
|
+
ferret gadget heron ion jackal kestrel lemur mantis newt orbit pelican quokka radar sprocket
|
|
43
|
+
tapir urchin vector wombat yak zephyr beacon cactus dolphin falcon gecko magnet pulsar""".split()
|
|
44
|
+
|
|
45
|
+
SCHEMA = "1" # bump when the cached shape of Job/Trial changes
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def home() -> str:
|
|
49
|
+
return paths.workspaces_dir()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
NAME_MAX = 48
|
|
53
|
+
# a name as slug() and _free() make it; one ending in a hyphen, or past 48 characters with a suffix, is how
|
|
54
|
+
# an earlier slug() could cut it, and is read still: its workspace is on disk and its owner made it
|
|
55
|
+
_NAME = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*-?")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def slug(label: str) -> str:
|
|
59
|
+
s = re.sub(r"[^a-z0-9]+", "-", (label or "").lower()).strip("-")
|
|
60
|
+
return s[:NAME_MAX].rstrip("-") # a cut on a separator leaves no hyphen at the end
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def random_name(taken=(), rng: random.Random | None = None) -> str:
|
|
64
|
+
rng = rng or random.Random()
|
|
65
|
+
taken = set(taken)
|
|
66
|
+
for _ in range(200):
|
|
67
|
+
name = f"{rng.choice(ADJECTIVES)}-{rng.choice(NOUNS)}"
|
|
68
|
+
if name not in taken:
|
|
69
|
+
return name
|
|
70
|
+
return f"{rng.choice(ADJECTIVES)}-{rng.choice(NOUNS)}-{rng.randrange(1000)}"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _now() -> str:
|
|
74
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class Workspace:
|
|
79
|
+
name: str
|
|
80
|
+
label: str
|
|
81
|
+
roots: list
|
|
82
|
+
created: str
|
|
83
|
+
opened: str
|
|
84
|
+
dir: str
|
|
85
|
+
job_roots: list = field(default_factory=list) # after imports: what parse walks
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def imports_dir(self) -> str:
|
|
89
|
+
return os.path.join(self.dir, "imported")
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def jobs_dir(self) -> str:
|
|
93
|
+
return os.path.join(self.dir, "jobs")
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def n_cached(self) -> int:
|
|
97
|
+
try:
|
|
98
|
+
return sum(1 for n in os.listdir(self.jobs_dir)
|
|
99
|
+
if n.endswith(".json") and n != "index.json")
|
|
100
|
+
except OSError:
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _path(home_dir: str, name: str) -> str:
|
|
105
|
+
return os.path.join(home_dir, name, "workspace.json")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _write_atomically(path: str, dump) -> None:
|
|
109
|
+
"""Write *path* whole or not at all: *dump* fills a temp file beside it,
|
|
110
|
+
which is then renamed over the target.
|
|
111
|
+
|
|
112
|
+
The temp name is this writer's own - process and thread - because two
|
|
113
|
+
requests writing the same file at once are ordinary here: the job page and
|
|
114
|
+
its /live poll both re-parse a job whose files keep changing. With one
|
|
115
|
+
shared `<name>.tmp`, the first rename took the file away and the second
|
|
116
|
+
raised FileNotFoundError, a 500 on the page. Whichever rename lands
|
|
117
|
+
last wins, and both wrote a complete file."""
|
|
118
|
+
tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp"
|
|
119
|
+
try:
|
|
120
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
121
|
+
dump(fh)
|
|
122
|
+
os.replace(tmp, path)
|
|
123
|
+
finally:
|
|
124
|
+
if os.path.exists(tmp):
|
|
125
|
+
os.unlink(tmp)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def save(ws: Workspace) -> None:
|
|
129
|
+
os.makedirs(ws.dir, exist_ok=True)
|
|
130
|
+
data = {k: v for k, v in asdict(ws).items() if k != "dir"}
|
|
131
|
+
_write_atomically(os.path.join(ws.dir, "workspace.json"), lambda fh: json.dump(data, fh, ensure_ascii=False, indent=1))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def load(home_dir: str, name: str) -> Workspace | None:
|
|
135
|
+
if not name or not _NAME.fullmatch(name):
|
|
136
|
+
return None
|
|
137
|
+
try:
|
|
138
|
+
with open(_path(home_dir, name), encoding="utf-8") as fh:
|
|
139
|
+
data = json.load(fh)
|
|
140
|
+
except (OSError, json.JSONDecodeError):
|
|
141
|
+
return None
|
|
142
|
+
if not isinstance(data, dict) or not data.get("name"):
|
|
143
|
+
return None
|
|
144
|
+
known = {f.name for f in fields(Workspace)} - {"dir"}
|
|
145
|
+
return Workspace(dir=os.path.join(home_dir, name),
|
|
146
|
+
**{k: v for k, v in data.items() if k in known})
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def list_all(home_dir: str) -> list:
|
|
150
|
+
"""Every workspace under *home_dir*, most recently opened first."""
|
|
151
|
+
try:
|
|
152
|
+
names = os.listdir(home_dir)
|
|
153
|
+
except OSError:
|
|
154
|
+
return []
|
|
155
|
+
found = [ws for ws in (load(home_dir, n) for n in sorted(names)) if ws]
|
|
156
|
+
return sorted(found, key=lambda w: w.opened, reverse=True)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def find_by_roots(home_dir: str, roots, besides: str = "") -> Workspace | None:
|
|
160
|
+
"""The workspace already pointing at exactly these roots, if any; the
|
|
161
|
+
most recently opened. ``besides``: a folder whose roots a workspace may
|
|
162
|
+
hold as well - the runs started from it, each added as it began - so a
|
|
163
|
+
restart with the same paths reopens it, its runs and all."""
|
|
164
|
+
want = sorted(os.path.realpath(r) for r in roots)
|
|
165
|
+
extra = os.path.join(os.path.realpath(besides), "") if besides else None
|
|
166
|
+
for ws in list_all(home_dir):
|
|
167
|
+
held = sorted(os.path.realpath(r) for r in ws.roots)
|
|
168
|
+
if held == want or (extra and [r for r in held if not r.startswith(extra)] == want):
|
|
169
|
+
return ws
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _free(name: str, taken) -> str:
|
|
174
|
+
"""*name*, or the first ``<name>-<n>`` no other workspace has, cut so that it stays a name."""
|
|
175
|
+
if name not in taken:
|
|
176
|
+
return name
|
|
177
|
+
return next(c for c in (f"{name[:NAME_MAX - len(str(i)) - 1].rstrip('-')}-{i}" for i in range(2, 1000)) if c not in taken)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def create(home_dir: str, roots, label: str = "") -> Workspace:
|
|
181
|
+
taken = {ws.name for ws in list_all(home_dir)}
|
|
182
|
+
name = slug(label)
|
|
183
|
+
name = _free(name, taken) if name else random_name(taken)
|
|
184
|
+
now = _now()
|
|
185
|
+
ws = Workspace(name=name, label=label.strip() or name, roots=list(roots),
|
|
186
|
+
created=now, opened=now, dir=os.path.join(home_dir, name))
|
|
187
|
+
save(ws)
|
|
188
|
+
return ws
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def rename(home_dir: str, ws: Workspace, label: str) -> Workspace:
|
|
192
|
+
"""*ws* called *label*, and named after it as a new workspace would be:
|
|
193
|
+
its directory moves to the label's slug (suffixed when another has it),
|
|
194
|
+
with everything in it - parsed jobs, imports, judgements - so a
|
|
195
|
+
workspace never carries one name on the page and another on disk. A
|
|
196
|
+
label with nothing to slug keeps the name it has. The roots it reads
|
|
197
|
+
stay where they are."""
|
|
198
|
+
label = (label or "").strip()
|
|
199
|
+
if not label:
|
|
200
|
+
return ws
|
|
201
|
+
name = slug(label)
|
|
202
|
+
if name and name != ws.name:
|
|
203
|
+
# a directory there that holds no workspace - left behind, half made - is taken too
|
|
204
|
+
taken = {w.name for w in list_all(home_dir)} | set(os.listdir(home_dir))
|
|
205
|
+
name = _free(name, taken - {ws.name})
|
|
206
|
+
new_dir = os.path.join(home_dir, name)
|
|
207
|
+
os.rename(ws.dir, new_dir)
|
|
208
|
+
ws.name, ws.dir = name, new_dir
|
|
209
|
+
ws.label = label
|
|
210
|
+
save(ws)
|
|
211
|
+
return ws
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def touch(ws: Workspace) -> None:
|
|
215
|
+
ws.opened = _now()
|
|
216
|
+
save(ws)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def delete(home_dir: str, name: str) -> bool:
|
|
220
|
+
"""Remove a workspace: its record, its parse cache and its converted
|
|
221
|
+
imports. The job roots it pointed at are not touched - they were never
|
|
222
|
+
its own. True when something was removed."""
|
|
223
|
+
ws = load(home_dir, name)
|
|
224
|
+
if ws is None:
|
|
225
|
+
return False
|
|
226
|
+
shutil.rmtree(ws.dir, ignore_errors=True)
|
|
227
|
+
return True
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def signature(job_dir: str) -> str:
|
|
232
|
+
"""What the parse of *job_dir* depends on: the files it reads, by size
|
|
233
|
+
and mtime. Cheap — a few hundred stats — and honest: a running job moves
|
|
234
|
+
its result.json files, so its signature moves with it."""
|
|
235
|
+
return _signatures(job_dir)[0]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _signatures(job_dir: str) -> tuple:
|
|
239
|
+
"""(the job's signature, {trial folder: its own part of it}): a trial whose
|
|
240
|
+
part is unchanged parses the same, so a running job re-reads only the
|
|
241
|
+
trials that moved (load_job), not all of them at each one that ends."""
|
|
242
|
+
parts = [f"schema={SCHEMA}/{parse.SCHEMA}"]
|
|
243
|
+
trials = {}
|
|
244
|
+
try:
|
|
245
|
+
names = sorted(os.listdir(job_dir))
|
|
246
|
+
except OSError:
|
|
247
|
+
return "", {}
|
|
248
|
+
|
|
249
|
+
def stat(path):
|
|
250
|
+
try:
|
|
251
|
+
st = os.stat(path)
|
|
252
|
+
except OSError:
|
|
253
|
+
return None
|
|
254
|
+
return f"{os.path.relpath(path, job_dir)}:{st.st_size}:{st.st_mtime_ns}"
|
|
255
|
+
|
|
256
|
+
for top in ("result.json", "config.json"):
|
|
257
|
+
s = stat(os.path.join(job_dir, top))
|
|
258
|
+
if s:
|
|
259
|
+
parts.append(s)
|
|
260
|
+
for n in names:
|
|
261
|
+
sub = os.path.join(job_dir, n)
|
|
262
|
+
if not os.path.isdir(sub):
|
|
263
|
+
continue
|
|
264
|
+
own = [s for s in (stat(os.path.join(sub, rel)) for rel in
|
|
265
|
+
("result.json", os.path.join("agent", "trajectory.json"), os.path.join("agent", "trace.json"))) if s]
|
|
266
|
+
parts += own
|
|
267
|
+
trials[sub] = "\n".join(own)
|
|
268
|
+
return hashlib.sha256("\n".join(parts).encode()).hexdigest(), trials
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _encode(obj):
|
|
272
|
+
if isinstance(obj, datetime):
|
|
273
|
+
return {"$dt": obj.isoformat()}
|
|
274
|
+
raise TypeError(f"not serialisable: {type(obj).__name__}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _decode(d):
|
|
278
|
+
if "$dt" in d and len(d) == 1:
|
|
279
|
+
return datetime.fromisoformat(d["$dt"])
|
|
280
|
+
return d
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def job_to_dict(job: parse.Job) -> dict:
|
|
284
|
+
return asdict(job)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def job_from_dict(data: dict) -> parse.Job:
|
|
288
|
+
trial_fields = {f.name for f in fields(parse.Trial)}
|
|
289
|
+
trials = []
|
|
290
|
+
for t in data.get("trials") or []:
|
|
291
|
+
t = {k: v for k, v in t.items() if k in trial_fields}
|
|
292
|
+
t["phases"] = {k: tuple(v) for k, v in (t.get("phases") or {}).items()}
|
|
293
|
+
trials.append(parse.Trial(**t))
|
|
294
|
+
job_fields = {f.name for f in fields(parse.Job)}
|
|
295
|
+
body = {k: v for k, v in data.items() if k in job_fields and k != "trials"}
|
|
296
|
+
job = parse.Job(**body)
|
|
297
|
+
job.trials = trials
|
|
298
|
+
return job
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _cache_file(ws: Workspace, job_dir: str) -> str:
|
|
302
|
+
key = hashlib.sha256(os.path.realpath(job_dir).encode()).hexdigest()[:20]
|
|
303
|
+
return os.path.join(ws.jobs_dir, f"{key}.json")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _index_path(ws: Workspace) -> str:
|
|
307
|
+
return os.path.join(ws.jobs_dir, "index.json")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def job_index(ws: Workspace) -> dict:
|
|
311
|
+
"""job dir → {job_id, n_trials}: what the rail lists without parsing."""
|
|
312
|
+
try:
|
|
313
|
+
with open(_index_path(ws), encoding="utf-8") as fh:
|
|
314
|
+
data = json.load(fh)
|
|
315
|
+
except (OSError, json.JSONDecodeError):
|
|
316
|
+
return {}
|
|
317
|
+
return data if isinstance(data, dict) else {}
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _note_in_index(ws: Workspace, job_dir: str, job: parse.Job, index: dict | None = None) -> bool:
|
|
321
|
+
"""The job's entry in the rail's index. With ``index`` (load_jobs, which
|
|
322
|
+
reads the file once and writes it once) the dict is changed in place and
|
|
323
|
+
True returned when it changed; alone, the file is read and written here."""
|
|
324
|
+
own = index is None
|
|
325
|
+
index = job_index(ws) if own else index
|
|
326
|
+
entry = {"job_id": job.job_id, "n_trials": len(job.trials), "dataset": job.dataset,
|
|
327
|
+
"agent": job.agent_label or job.agent_name}
|
|
328
|
+
if index.get(job_dir) == entry:
|
|
329
|
+
return False
|
|
330
|
+
index[job_dir] = entry
|
|
331
|
+
if own:
|
|
332
|
+
_write_index(ws, index)
|
|
333
|
+
return True
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _write_index(ws: Workspace, index: dict) -> None:
|
|
337
|
+
os.makedirs(ws.jobs_dir, exist_ok=True)
|
|
338
|
+
_write_atomically(_index_path(ws), lambda fh: json.dump(index, fh, ensure_ascii=False, indent=1))
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# Jobs already decoded in this process, by cache file: (signature, Job). A page
|
|
342
|
+
# that loads the workspace then skips the JSON decode and the rebuild of every
|
|
343
|
+
# unchanged job. A Job is never changed once loaded (parse.split_agents and the
|
|
344
|
+
# compare page make new ones with dataclasses.replace).
|
|
345
|
+
_DECODED: dict = {}
|
|
346
|
+
_DECODED_LOCK = threading.Lock()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple:
|
|
350
|
+
"""(Job, from_cache). Parses only when the signature changed. ``index``:
|
|
351
|
+
the rail's index as load_jobs holds it, noted in place."""
|
|
352
|
+
sig, trial_sigs = _signatures(job_dir)
|
|
353
|
+
path = _cache_file(ws, job_dir)
|
|
354
|
+
with _DECODED_LOCK:
|
|
355
|
+
known = _DECODED.get(path)
|
|
356
|
+
if sig:
|
|
357
|
+
if known is not None and known[0] == sig and known[1].dir == job_dir:
|
|
358
|
+
_note_in_index(ws, job_dir, known[1], index)
|
|
359
|
+
return known[1], True
|
|
360
|
+
try:
|
|
361
|
+
with open(path, encoding="utf-8") as fh:
|
|
362
|
+
cached = json.load(fh, object_hook=_decode)
|
|
363
|
+
# "agents": a cache from before split_agents knows nothing of a job's several agents
|
|
364
|
+
if cached.get("signature") == sig and cached.get("dir") == job_dir and "agents" in cached.get("job", {}):
|
|
365
|
+
job = job_from_dict(cached["job"])
|
|
366
|
+
with _DECODED_LOCK:
|
|
367
|
+
_DECODED[path] = (sig, job, trial_sigs)
|
|
368
|
+
_note_in_index(ws, job_dir, job, index)
|
|
369
|
+
return job, True
|
|
370
|
+
except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError):
|
|
371
|
+
pass
|
|
372
|
+
reuse = None
|
|
373
|
+
if known is not None and known[1].dir == job_dir:
|
|
374
|
+
before = {t.dir: t for t in known[1].trials}
|
|
375
|
+
old_sigs = known[2]
|
|
376
|
+
# a trial whose own files are as they were parses the same: kept, not read again
|
|
377
|
+
reuse = lambda sub: before.get(sub) if old_sigs.get(sub) and old_sigs.get(sub) == trial_sigs.get(sub) else None
|
|
378
|
+
job = parse.parse_job(job_dir, reuse=reuse)
|
|
379
|
+
if sig:
|
|
380
|
+
os.makedirs(ws.jobs_dir, exist_ok=True)
|
|
381
|
+
_write_atomically(path, lambda fh: json.dump({"signature": sig, "dir": job_dir, "cached": _now(),
|
|
382
|
+
"job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False))
|
|
383
|
+
with _DECODED_LOCK:
|
|
384
|
+
_DECODED[path] = (sig, job, trial_sigs)
|
|
385
|
+
_note_in_index(ws, job_dir, job, index)
|
|
386
|
+
return job, False
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def load_jobs(ws: Workspace, roots) -> list:
|
|
390
|
+
"""Every job under *roots*, each from the cache when its files are
|
|
391
|
+
unchanged. Same order and de-duplication as ``parse.load_all``."""
|
|
392
|
+
# the rail's index read once and written once, not once per job: re-reading it for
|
|
393
|
+
# each made a workspace of n jobs cost n parses of an index of n entries
|
|
394
|
+
index = job_index(ws)
|
|
395
|
+
before = dict(index)
|
|
396
|
+
jobs = [load_job(ws, d, index)[0] for d in parse.find_job_dirs(roots)]
|
|
397
|
+
if index != before:
|
|
398
|
+
_write_index(ws, index)
|
|
399
|
+
return parse.split_agents(jobs)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def clear_cache(ws: Workspace) -> int:
|
|
403
|
+
"""Drop every cached parse (the index too), this process's decoded jobs
|
|
404
|
+
included. Returns how many jobs."""
|
|
405
|
+
with _DECODED_LOCK:
|
|
406
|
+
for path in [p for p in _DECODED if os.path.dirname(p) == ws.jobs_dir]:
|
|
407
|
+
del _DECODED[path]
|
|
408
|
+
try:
|
|
409
|
+
names = [n for n in os.listdir(ws.jobs_dir) if n.endswith(".json")]
|
|
410
|
+
except OSError:
|
|
411
|
+
return 0
|
|
412
|
+
for n in names:
|
|
413
|
+
try:
|
|
414
|
+
os.remove(os.path.join(ws.jobs_dir, n))
|
|
415
|
+
except OSError:
|
|
416
|
+
pass
|
|
417
|
+
return len([n for n in names if n != "index.json"])
|
nullius/workspace_cli.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""List the workspaces from a shell: what /workspaces shows.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.workspace_cli list # under WORKSPACES_DIR, or workspaces/
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.workspace_cli list --home DIR
|
|
5
|
+
PYTHONPATH=app python3 -m nullius.workspace_cli list --json
|
|
6
|
+
|
|
7
|
+
A workspace is a named set of job roots and what the app read from them
|
|
8
|
+
(``nullius.workspace``). ``list`` reads each ``workspace.json`` under the
|
|
9
|
+
home directory the way the page does (``workspace.list_all``: most recently
|
|
10
|
+
opened first) and prints, per workspace, its name, its label, its roots, the
|
|
11
|
+
runs its parse cache holds (the page's *Cached runs*, and their ids from the
|
|
12
|
+
cache's index, both known without parsing anything) and when it was last
|
|
13
|
+
opened. It opens nothing: no workspace becomes active, no root is walked,
|
|
14
|
+
no import converted, no cache touched.
|
|
15
|
+
|
|
16
|
+
The home is ``--home``, else ``WORKSPACES_DIR``, else ``workspaces/`` in the
|
|
17
|
+
working directory — the app's own rule (``workspace.home``). Renaming and
|
|
18
|
+
deleting stay the page's: a delete removes the cache and the converted
|
|
19
|
+
imports behind a confirmation, and nothing here writes.
|
|
20
|
+
|
|
21
|
+
Reading only. The exit codes are the contract in ``docs/cli.md``.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import sys
|
|
29
|
+
|
|
30
|
+
from nullius import workspace
|
|
31
|
+
|
|
32
|
+
EXIT_OK = 0
|
|
33
|
+
EXIT_NOT_FOUND = 3
|
|
34
|
+
|
|
35
|
+
EPILOG = """\
|
|
36
|
+
examples:
|
|
37
|
+
PYTHONPATH=app python3 -m nullius.workspace_cli list
|
|
38
|
+
WORKSPACES_DIR=/srv/nullius/workspaces PYTHONPATH=app python3 -m nullius.workspace_cli list --json
|
|
39
|
+
PYTHONPATH=app python3 -m nullius.workspace_cli list --home /tmp/ws
|
|
40
|
+
|
|
41
|
+
exit codes:
|
|
42
|
+
0 done
|
|
43
|
+
1 unexpected error (a traceback: report it)
|
|
44
|
+
2 usage: a bad flag, a missing argument
|
|
45
|
+
3 input not found: no such home directory, no workspace under it
|
|
46
|
+
4 the operation could not be completed
|
|
47
|
+
With --json, stdout is exactly one JSON document; everything else goes to stderr.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def row(ws: workspace.Workspace) -> dict:
|
|
52
|
+
"""One workspace as the page's table shows it, plus the job ids its
|
|
53
|
+
cache index names — the rail's list, known without a parse."""
|
|
54
|
+
index = workspace.job_index(ws)
|
|
55
|
+
return {
|
|
56
|
+
"name": ws.name,
|
|
57
|
+
"label": ws.label,
|
|
58
|
+
"dir": ws.dir,
|
|
59
|
+
"roots": list(ws.roots),
|
|
60
|
+
"job_roots": list(ws.job_roots),
|
|
61
|
+
"n_cached": ws.n_cached,
|
|
62
|
+
"jobs": [{"dir": job_dir, "job_id": entry.get("job_id"), "n_trials": entry.get("n_trials"),
|
|
63
|
+
"dataset": entry.get("dataset"), "agent": entry.get("agent")}
|
|
64
|
+
for job_dir, entry in index.items() if isinstance(entry, dict)],
|
|
65
|
+
"created": ws.created,
|
|
66
|
+
"opened": ws.opened,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _emit(doc) -> None:
|
|
71
|
+
json.dump(doc, sys.stdout, ensure_ascii=False, indent=1)
|
|
72
|
+
sys.stdout.write("\n")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _list(args) -> int:
|
|
76
|
+
home = args.home or workspace.home()
|
|
77
|
+
if not os.path.isdir(home):
|
|
78
|
+
print(f"no such directory: {home}", file=sys.stderr)
|
|
79
|
+
return EXIT_NOT_FOUND
|
|
80
|
+
found = workspace.list_all(home)
|
|
81
|
+
if not found:
|
|
82
|
+
print(f"no workspace found under {home}", file=sys.stderr)
|
|
83
|
+
return EXIT_NOT_FOUND
|
|
84
|
+
if args.json:
|
|
85
|
+
_emit([row(ws) for ws in found])
|
|
86
|
+
return EXIT_OK
|
|
87
|
+
width = max(len(ws.name) for ws in found)
|
|
88
|
+
for ws in found:
|
|
89
|
+
r = row(ws)
|
|
90
|
+
opened = (ws.opened or "")[:16].replace("T", " ")
|
|
91
|
+
print(f"{ws.name:<{width}} {ws.label}")
|
|
92
|
+
print(f"{'':<{width}} cached runs {r['n_cached']} · opened {opened or '—'}")
|
|
93
|
+
for root in ws.roots:
|
|
94
|
+
print(f"{'':<{width}} root {root}")
|
|
95
|
+
for job in r["jobs"]:
|
|
96
|
+
print(f"{'':<{width}} job {job['job_id']} ({job['n_trials']} trial(s))")
|
|
97
|
+
return EXIT_OK
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main(argv=None) -> int:
|
|
101
|
+
ap = argparse.ArgumentParser(
|
|
102
|
+
prog="python3 -m nullius.workspace_cli", description=__doc__.splitlines()[0],
|
|
103
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
104
|
+
sub = ap.add_subparsers(dest="command", metavar="COMMAND")
|
|
105
|
+
ls = sub.add_parser("list", help="the workspaces under the home directory, most recently opened first",
|
|
106
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
107
|
+
ls.add_argument("--home", metavar="DIR", default="",
|
|
108
|
+
help="the workspaces directory (default: WORKSPACES_DIR, else workspaces/)")
|
|
109
|
+
ls.add_argument("--json", action="store_true",
|
|
110
|
+
help="one JSON array, a row per workspace, instead of the listing")
|
|
111
|
+
ls.set_defaults(run=_list)
|
|
112
|
+
args = ap.parse_args(argv)
|
|
113
|
+
if not args.command:
|
|
114
|
+
ap.error("a command is required: list")
|
|
115
|
+
return args.run(args)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__": # pragma: no cover - entry point
|
|
119
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nullius-eval
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An evaluation framework for AI agents. Bring any benchmark, agent or capability, run it on your machines, and inspect the evidence behind every score. Bring anything, run it, prove it.
|
|
5
|
+
Maintainer-email: ThinkFlowLab <nullius.thinkflowlab@proton.me>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://thinkflowlab.ai/nullius/
|
|
8
|
+
Project-URL: Documentation, https://thinkflowlab.ai/nullius/docs/
|
|
9
|
+
Project-URL: Repository, https://github.com/thinkflowlab/nullius
|
|
10
|
+
Project-URL: Issues, https://github.com/thinkflowlab/nullius/issues
|
|
11
|
+
Project-URL: Releases, https://github.com/thinkflowlab/nullius/releases
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
License-File: app/static/fonts/dmsans-OFL.txt
|
|
20
|
+
License-File: app/static/fonts/ibmplexmono-OFL.txt
|
|
21
|
+
License-File: app/static/fonts/instrumentserif-OFL.txt
|
|
22
|
+
Requires-Dist: flask>=3.1.3
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# Nullius
|
|
26
|
+
|
|
27
|
+
Bring anything. Run it. Prove it.
|
|
28
|
+
|
|
29
|
+
Nullius runs AI evaluations on your machines and opens each score into the
|
|
30
|
+
evidence behind it, so you know whether the agent failed or the benchmark did.\
|
|
31
|
+
Add your own benchmark, agent or capability with a single prompt and iterate
|
|
32
|
+
faster.
|
|
33
|
+
|
|
34
|
+
## Install and launch
|
|
35
|
+
|
|
36
|
+
Install the latest version with [uv](https://docs.astral.sh/uv/):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
uv tool install nullius-eval
|
|
40
|
+
nullius serve
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Open [127.0.0.1:5057](http://127.0.0.1:5057). The package name is `nullius-eval`;
|
|
44
|
+
the application and command are `nullius`.
|
|
45
|
+
|
|
46
|
+
Python 3.12 or later is required. Reading existing results needs no model key.
|
|
47
|
+
Running benchmarks requires the appropriate provider credentials and benchmark
|
|
48
|
+
prerequisites; Docker is needed for Harbor and the standard's container backend.
|
|
49
|
+
|
|
50
|
+
Release validation targets Linux with Python 3.12. The wheel is a Python package with
|
|
51
|
+
a local web UI, not a standalone desktop installer.
|
|
52
|
+
|
|
53
|
+
## What it does
|
|
54
|
+
|
|
55
|
+
- Bring benchmarks, agents and capabilities into a catalogue and match them by
|
|
56
|
+
the ports they have proven.
|
|
57
|
+
- Build evaluation recipes, review their prerequisites, then run locally,
|
|
58
|
+
over SSH or across several machines.
|
|
59
|
+
- Inspect trial outcomes, verifier results, model calls, conversations and
|
|
60
|
+
artifacts; compare the shared tasks across runs.
|
|
61
|
+
- Check benchmarks with controls and inspect results with optional model
|
|
62
|
+
judgement.
|
|
63
|
+
|
|
64
|
+
Settings, catalogue additions and runs are stored outside the installed
|
|
65
|
+
package. `nullius paths` shows their locations, and reinstalling the tool
|
|
66
|
+
preserves them.
|
|
67
|
+
|
|
68
|
+
## Updating
|
|
69
|
+
|
|
70
|
+
Update to the latest version with:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
uv tool upgrade nullius-eval
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
To select a specific version, use `uv tool install nullius-eval==0.1.0`.
|
|
77
|
+
Add `--force` when replacing an existing installation. Explicit version pins
|
|
78
|
+
are retained by `uv tool upgrade`; use `uv tool install --force nullius-eval`
|
|
79
|
+
to return to the latest version without a pin.
|
|
80
|
+
|
|
81
|
+
If you used an earlier private build installed under the distribution name
|
|
82
|
+
`nullius`, remove that tool with `uv tool uninstall nullius` before installing
|
|
83
|
+
`nullius-eval`. The user storage remains in place.
|
|
84
|
+
|
|
85
|
+
## Links
|
|
86
|
+
|
|
87
|
+
- [Documentation](https://thinkflowlab.ai/nullius/docs/)
|
|
88
|
+
- [Quickstart](https://thinkflowlab.ai/nullius/docs/getting-started/)
|
|
89
|
+
- [Repository](https://github.com/thinkflowlab/nullius)
|
|
90
|
+
- [Report an issue](https://github.com/thinkflowlab/nullius/issues)
|
|
91
|
+
|
|
92
|
+
GitHub access follows the repository's permissions. Nullius is licensed under
|
|
93
|
+
Apache-2.0; bundled fonts retain their own included licences.
|