nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""ATIF trajectories from any producer: standalone documents in, one job out.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.importers.cli trajectories/ -o jobs_imported
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.importers.cli agent/trajectory.json -o jobs_imported
|
|
5
|
+
|
|
6
|
+
An ATIF document is the file Harbor's agents write as ``agent/trajectory.json``
|
|
7
|
+
and the reader already opens inside a trial (``nullius.traces.trajectory``).
|
|
8
|
+
On its own — dropped on the import picker, or a directory of them with no
|
|
9
|
+
``result.json`` beside — nothing recognised it. This does, by content: a JSON
|
|
10
|
+
object whose ``schema_version`` reads ``ATIF-v1.x`` and which carries
|
|
11
|
+
``steps``. The producer's name, the file's name and the directory it sits in
|
|
12
|
+
are not consulted, so a trajectory from claude-code, opencode or
|
|
13
|
+
any other adopter of the schema is read the same way, and two copies of one
|
|
14
|
+
document normalise identically wherever they were found.
|
|
15
|
+
|
|
16
|
+
**Supported schema versions**: ``ATIF-v1.0`` through ``ATIF-v1.7``
|
|
17
|
+
(``SUPPORTED_VERSIONS``) — the versions Nullius already parses inside a job,
|
|
18
|
+
as written by Harbor's agents (``final_metrics``, per-step
|
|
19
|
+
``metrics``, the ``extra`` usage slot). A document of any other version is
|
|
20
|
+
refused by name rather than read as though it were one of these.
|
|
21
|
+
|
|
22
|
+
The app walks Harbor's shape, so this converts: one directory (or one file) becomes one job, each document
|
|
23
|
+
a trial directory holding the document *copied byte for byte* as
|
|
24
|
+
``agent/trajectory.json``. Nothing is parsed twice — ``trajectory.load`` is
|
|
25
|
+
the one reader, here and on the trial page — and nothing is re-measured:
|
|
26
|
+
|
|
27
|
+
- **No benchmark outcome is invented.** A trajectory records what an agent
|
|
28
|
+
did, not whether it was right. The trial carries no ``verifier_result``,
|
|
29
|
+
no reward and no ``verifier/`` marker, so it reads as ``unknown`` — never
|
|
30
|
+
a score, never a solve rate — and stays out of every rate.
|
|
31
|
+
- **Usage and timing are the document's own, or absent.** ``agent_result``
|
|
32
|
+
copies ``final_metrics`` (prompt, completion, cached, cost) and is null
|
|
33
|
+
where the document is silent; the trial window is the span of the step
|
|
34
|
+
timestamps, and null when no step carries one. Nothing is zeroed to fill
|
|
35
|
+
a column: "unmeasured" and "zero" are different claims.
|
|
36
|
+
- **Provenance is the document's.** ``agent_info`` comes from the ATIF
|
|
37
|
+
``agent`` block (name, version, model_name), falling back to the first
|
|
38
|
+
step's ``model_name`` for the model; the schema version, session id and
|
|
39
|
+
the source file's path and digest are kept on the trial record under
|
|
40
|
+
``source_document``.
|
|
41
|
+
- **The recorded command is the import.** No harness ran here, so the line
|
|
42
|
+
the job page offers to copy is the one that reproduces this job
|
|
43
|
+
directory, not a ``harbor run`` that never happened.
|
|
44
|
+
|
|
45
|
+
A tree this claims but cannot read is refused before anything is written,
|
|
46
|
+
with a ``SystemExit`` whose message names the file (relative to the tree
|
|
47
|
+
given) and what is wrong — a document that is not JSON, one without
|
|
48
|
+
``steps``, one of an unsupported version — and is redacted: a credential in
|
|
49
|
+
a bad file's name or content must not travel into the error. A ``.json``
|
|
50
|
+
that is well-formed but not ATIF at all (a ``trace.json``, a
|
|
51
|
+
``spans.meta.json``) is simply not one of ours and is passed over.
|
|
52
|
+
|
|
53
|
+
What this does *not* claim: a Harbor job or trial (a ``result.json`` on the
|
|
54
|
+
walk). Those trajectories belong to the record around them, whose accounting
|
|
55
|
+
is the reader's and must not be shadowed by a second, thinner reading of the same
|
|
56
|
+
file.
|
|
57
|
+
|
|
58
|
+
Re-converting is idempotent: trial names are
|
|
59
|
+
deterministic (the document's path relative to the tree, a bare
|
|
60
|
+
``trajectory.json`` named after its directory), every file is rewritten in
|
|
61
|
+
place, and a trial the tree no longer holds is removed.
|
|
62
|
+
"""
|
|
63
|
+
from __future__ import annotations
|
|
64
|
+
|
|
65
|
+
import hashlib
|
|
66
|
+
import json
|
|
67
|
+
import os
|
|
68
|
+
import re
|
|
69
|
+
import shlex
|
|
70
|
+
import shutil
|
|
71
|
+
from datetime import timezone
|
|
72
|
+
|
|
73
|
+
from nullius.redact import redact
|
|
74
|
+
from nullius.traces.trajectory import parse_ts
|
|
75
|
+
|
|
76
|
+
NAME = "atif"
|
|
77
|
+
LABEL = "ATIF"
|
|
78
|
+
|
|
79
|
+
# The schema versions this reads — the ones `trajectory.py` is known to
|
|
80
|
+
# parse inside a Harbor job. Anchoring on the list is what keeps detect()
|
|
81
|
+
# from claiming a future shape and reading it as one of these.
|
|
82
|
+
SUPPORTED_VERSIONS = tuple(f"ATIF-v1.{minor}" for minor in range(8))
|
|
83
|
+
|
|
84
|
+
# What makes a JSON object an ATIF document at all, whatever its version:
|
|
85
|
+
# the schema tag. Shape (steps present, a list) is checked when read, so a
|
|
86
|
+
# partial document is refused by name rather than passed over as foreign.
|
|
87
|
+
_VERSION_RE = re.compile(r"^ATIF-v\d+\.\d+$")
|
|
88
|
+
|
|
89
|
+
# A file that is not JSON is only *ours* to refuse when it says it is an
|
|
90
|
+
# ATIF document; sniffed in the head of the file, since it cannot be parsed.
|
|
91
|
+
_MARKER = b'"ATIF-v'
|
|
92
|
+
_SNIFF_BYTES = 4096
|
|
93
|
+
|
|
94
|
+
# Files that mark a directory as part of a record another reader owns.
|
|
95
|
+
_OWNED_BY = ("result.json",)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_atif(doc) -> bool:
|
|
100
|
+
return (isinstance(doc, dict)
|
|
101
|
+
and isinstance(doc.get("schema_version"), str)
|
|
102
|
+
and bool(_VERSION_RE.match(doc["schema_version"])))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _mentions_atif(path: str) -> bool:
|
|
106
|
+
try:
|
|
107
|
+
with open(path, "rb") as fh:
|
|
108
|
+
return _MARKER in fh.read(_SNIFF_BYTES)
|
|
109
|
+
except OSError:
|
|
110
|
+
return False
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _read(path: str, shown: str) -> dict:
|
|
114
|
+
"""The document at *path*, or a SystemExit naming what is wrong with it.
|
|
115
|
+
|
|
116
|
+
The message is what the CLI prints and the picker shows, so it is
|
|
117
|
+
redacted here, at construction, whichever branch built it.
|
|
118
|
+
"""
|
|
119
|
+
try:
|
|
120
|
+
with open(path, encoding="utf-8") as fh:
|
|
121
|
+
doc = json.load(fh)
|
|
122
|
+
except OSError as e:
|
|
123
|
+
raise SystemExit(redact(f"{shown}: cannot be read ({e.strerror})")) from None
|
|
124
|
+
except ValueError as e:
|
|
125
|
+
raise SystemExit(redact(f"{shown}: not JSON ({e})")) from None
|
|
126
|
+
if not _is_atif(doc):
|
|
127
|
+
raise SystemExit(redact(f"{shown}: not an ATIF document"))
|
|
128
|
+
version = doc["schema_version"]
|
|
129
|
+
if version not in SUPPORTED_VERSIONS:
|
|
130
|
+
raise SystemExit(redact(
|
|
131
|
+
f"{shown}: unsupported ATIF schema_version {version!r} "
|
|
132
|
+
f"(supported: {SUPPORTED_VERSIONS[0]} through {SUPPORTED_VERSIONS[-1]})"))
|
|
133
|
+
if "steps" not in doc:
|
|
134
|
+
raise SystemExit(redact(f"{shown}: no steps (a partial document)"))
|
|
135
|
+
if not isinstance(doc["steps"], list):
|
|
136
|
+
raise SystemExit(redact(f"{shown}: steps is not a list"))
|
|
137
|
+
return doc
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _claims(path: str) -> bool:
|
|
141
|
+
"""Is this file ours — an ATIF document, or a broken file that says it is?"""
|
|
142
|
+
try:
|
|
143
|
+
with open(path, encoding="utf-8") as fh:
|
|
144
|
+
doc = json.load(fh)
|
|
145
|
+
except OSError:
|
|
146
|
+
return False
|
|
147
|
+
except ValueError:
|
|
148
|
+
return _mentions_atif(path)
|
|
149
|
+
return _is_atif(doc)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def find_documents(root: str) -> list[str]:
|
|
153
|
+
"""Every file under *root* this importer claims, sorted; *root* itself
|
|
154
|
+
when it is one file. Empty when a ``result.json`` is
|
|
155
|
+
seen on the walk — that tree is a record another reader owns."""
|
|
156
|
+
if os.path.isfile(root):
|
|
157
|
+
return [root] if _claims(root) else []
|
|
158
|
+
found = []
|
|
159
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
160
|
+
dirnames.sort()
|
|
161
|
+
if any(name in filenames for name in _OWNED_BY):
|
|
162
|
+
return []
|
|
163
|
+
for name in sorted(filenames):
|
|
164
|
+
path = os.path.join(dirpath, name)
|
|
165
|
+
if name.endswith(".json") and _claims(path):
|
|
166
|
+
found.append(path)
|
|
167
|
+
return found
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def detect(path: str) -> bool:
|
|
171
|
+
"""Does *path* hold standalone ATIF documents (or is it one)?"""
|
|
172
|
+
return bool(find_documents(path))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def check(path: str) -> None:
|
|
176
|
+
"""Read every document the way `convert` would, writing nothing; raises
|
|
177
|
+
``SystemExit`` (redacted) for the first thing `convert` would refuse."""
|
|
178
|
+
_read_all(path)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def trees(root: str) -> list[str]:
|
|
182
|
+
"""One directory, one job, whatever its depth — the CLI's reading of
|
|
183
|
+
*root*, shared by the picker. The documents found under it are
|
|
184
|
+
that job's trials, named by their place in the tree (`_trial_name`), so
|
|
185
|
+
a nested directory of trajectories is one job of several trials rather
|
|
186
|
+
than a job per child."""
|
|
187
|
+
return [root] if detect(root) else []
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _shown(path: str, root: str) -> str:
|
|
191
|
+
"""How a file is named in a message: relative to the tree given."""
|
|
192
|
+
if os.path.isfile(root):
|
|
193
|
+
return os.path.basename(path)
|
|
194
|
+
return os.path.relpath(path, root)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _read_all(root: str) -> list[tuple[str, dict]]:
|
|
198
|
+
paths = find_documents(root)
|
|
199
|
+
if not paths:
|
|
200
|
+
raise SystemExit(redact(
|
|
201
|
+
f"no ATIF trajectories (schema_version {SUPPORTED_VERSIONS[0]} through "
|
|
202
|
+
f"{SUPPORTED_VERSIONS[-1]}) found in {root}"))
|
|
203
|
+
return [(path, _read(path, _shown(path, root))) for path in paths]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _trial_name(path: str, root: str) -> str:
|
|
208
|
+
"""The document's place in the tree, flattened for a directory name.
|
|
209
|
+
|
|
210
|
+
A bare ``trajectory.json`` is named after the directory holding it, so
|
|
211
|
+
``claude-code/trajectory.json`` and ``claude-code.json`` are the same
|
|
212
|
+
trial under either layout. A single file, or a ``trajectory.json`` at
|
|
213
|
+
the root itself, takes the file's own stem.
|
|
214
|
+
"""
|
|
215
|
+
if os.path.isfile(root):
|
|
216
|
+
rel = os.path.basename(path)
|
|
217
|
+
else:
|
|
218
|
+
rel = os.path.relpath(path, root)
|
|
219
|
+
stem = rel[:-len(".json")] if rel.endswith(".json") else rel
|
|
220
|
+
parts = [p for p in stem.split(os.sep) if p and p != "."]
|
|
221
|
+
if len(parts) > 1 and parts[-1] == "trajectory":
|
|
222
|
+
parts.pop()
|
|
223
|
+
return "__".join(parts) or "trajectory"
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _instant(stamp: str) -> float:
|
|
227
|
+
"""A stamp as seconds since the epoch, a naive one read as UTC: what the
|
|
228
|
+
job's first start and last end are chosen by."""
|
|
229
|
+
when = parse_ts(stamp)
|
|
230
|
+
if when is None:
|
|
231
|
+
return float("inf")
|
|
232
|
+
return (when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when).timestamp()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _window(doc: dict) -> tuple[str | None, str | None]:
|
|
236
|
+
"""The first and last step timestamps that parse, in step order and as
|
|
237
|
+
recorded; None when no step carries a clock — the trial then has no
|
|
238
|
+
duration, not a zero one. Step order rather than min/max: the document's
|
|
239
|
+
ordering is preserved, and two clocks of different kinds (one naive, one
|
|
240
|
+
zoned) are never compared."""
|
|
241
|
+
stamps = [s.get("timestamp") for s in doc["steps"]
|
|
242
|
+
if isinstance(s, dict) and parse_ts(s.get("timestamp")) is not None]
|
|
243
|
+
if not stamps:
|
|
244
|
+
return None, None
|
|
245
|
+
return stamps[0], stamps[-1]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _agent_info(doc: dict) -> dict:
|
|
249
|
+
agent = doc.get("agent") if isinstance(doc.get("agent"), dict) else {}
|
|
250
|
+
model = agent.get("model_name") or next(
|
|
251
|
+
(s.get("model_name") for s in doc["steps"]
|
|
252
|
+
if isinstance(s, dict) and s.get("model_name")), "")
|
|
253
|
+
return {
|
|
254
|
+
"name": str(agent.get("name") or ""),
|
|
255
|
+
"version": str(agent.get("version") or ""),
|
|
256
|
+
"model_info": {"name": str(model or "")},
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _usage(doc: dict) -> dict:
|
|
261
|
+
"""``final_metrics`` in the fields the reader sums — null where absent."""
|
|
262
|
+
metrics = doc.get("final_metrics") if isinstance(doc.get("final_metrics"), dict) else {}
|
|
263
|
+
|
|
264
|
+
def number(key):
|
|
265
|
+
value = metrics.get(key)
|
|
266
|
+
return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
|
267
|
+
|
|
268
|
+
return {
|
|
269
|
+
"n_input_tokens": number("total_prompt_tokens"),
|
|
270
|
+
"n_output_tokens": number("total_completion_tokens"),
|
|
271
|
+
"n_cache_tokens": number("total_cached_tokens"),
|
|
272
|
+
"cost_usd": number("total_cost_usd"),
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _sha256(path: str) -> str:
|
|
277
|
+
digest = hashlib.sha256()
|
|
278
|
+
with open(path, "rb") as fh:
|
|
279
|
+
for chunk in iter(lambda: fh.read(1 << 16), b""):
|
|
280
|
+
digest.update(chunk)
|
|
281
|
+
return digest.hexdigest()
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# ── one document = one trial ────────────────────────────────────────────────
|
|
285
|
+
|
|
286
|
+
def convert_document(path: str, doc: dict, trial: str, job_dir: str) -> dict:
|
|
287
|
+
"""Write one document as a Harbor-shaped trial directory. Returns a summary."""
|
|
288
|
+
trial_dir = os.path.join(job_dir, trial)
|
|
289
|
+
agent_dir = os.path.join(trial_dir, "agent")
|
|
290
|
+
os.makedirs(agent_dir, exist_ok=True)
|
|
291
|
+
|
|
292
|
+
started, finished = _window(doc)
|
|
293
|
+
result = {
|
|
294
|
+
"trial_name": trial,
|
|
295
|
+
"task_name": f"{NAME}/{trial}",
|
|
296
|
+
# The harness, not a dataset: the benchmark readers stay unengaged,
|
|
297
|
+
# and with no report and no reward the outcome is `unknown`.
|
|
298
|
+
"source": NAME,
|
|
299
|
+
"agent_info": _agent_info(doc),
|
|
300
|
+
"exception_info": {},
|
|
301
|
+
"verifier_result": {},
|
|
302
|
+
"started_at": started,
|
|
303
|
+
"finished_at": finished,
|
|
304
|
+
"environment_setup": None,
|
|
305
|
+
"agent_setup": None,
|
|
306
|
+
# The document is the agent's own window; what surrounded it (image
|
|
307
|
+
# pull, setup, a verifier) was never recorded here.
|
|
308
|
+
"agent_execution": (
|
|
309
|
+
{"started_at": started, "finished_at": finished} if started else None),
|
|
310
|
+
"verifier": None,
|
|
311
|
+
"agent_result": _usage(doc),
|
|
312
|
+
"source_document": {
|
|
313
|
+
"path": os.path.abspath(path),
|
|
314
|
+
"sha256": _sha256(path),
|
|
315
|
+
"schema_version": doc["schema_version"],
|
|
316
|
+
"session_id": doc.get("session_id"),
|
|
317
|
+
},
|
|
318
|
+
}
|
|
319
|
+
with open(os.path.join(trial_dir, "result.json"), "w", encoding="utf-8") as fh:
|
|
320
|
+
json.dump(result, fh, indent=2)
|
|
321
|
+
|
|
322
|
+
# Byte for byte: the trial page reads what the producer wrote, and a
|
|
323
|
+
# digest of the copy is a digest of the source.
|
|
324
|
+
shutil.copyfile(path, os.path.join(agent_dir, "trajectory.json"))
|
|
325
|
+
|
|
326
|
+
return {"trial_dir": trial, "started_at": started, "finished_at": finished,
|
|
327
|
+
"agent_info": result["agent_info"]}
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _write_job(docs: list, root: str, out_root: str, job_name: str, command: str) -> str:
|
|
332
|
+
"""The read documents, as one job directory of one trial each."""
|
|
333
|
+
job_dir = os.path.join(out_root, job_name)
|
|
334
|
+
os.makedirs(job_dir, exist_ok=True)
|
|
335
|
+
|
|
336
|
+
summaries = [convert_document(path, doc, _trial_name(path, root), job_dir)
|
|
337
|
+
for path, doc in docs]
|
|
338
|
+
# Derived output this importer wholly owns: a trial the tree no longer
|
|
339
|
+
# holds is stale and would keep being counted.
|
|
340
|
+
produced = {s["trial_dir"] for s in summaries}
|
|
341
|
+
for name in os.listdir(job_dir):
|
|
342
|
+
sub = os.path.join(job_dir, name)
|
|
343
|
+
if os.path.isdir(sub) and name not in produced:
|
|
344
|
+
shutil.rmtree(sub)
|
|
345
|
+
|
|
346
|
+
# ordered as instants, not as text: one stamp with a zone and one without sort apart as strings
|
|
347
|
+
starts = sorted((s["started_at"] for s in summaries if s["started_at"]), key=_instant)
|
|
348
|
+
ends = sorted((s["finished_at"] for s in summaries if s["finished_at"]), key=_instant)
|
|
349
|
+
with open(os.path.join(job_dir, "result.json"), "w", encoding="utf-8") as fh:
|
|
350
|
+
json.dump({
|
|
351
|
+
"started_at": starts[0] if starts else None,
|
|
352
|
+
"finished_at": ends[-1] if ends else None,
|
|
353
|
+
"n_total_trials": len(summaries),
|
|
354
|
+
"stats": {"n_errored_trials": 0, "n_running_trials": 0,
|
|
355
|
+
"n_pending_trials": 0, "n_retries": 0},
|
|
356
|
+
}, fh, indent=2)
|
|
357
|
+
|
|
358
|
+
# The first document's producer names the job; a tree that mixes
|
|
359
|
+
# producers still shows each trial's own under agent_info.
|
|
360
|
+
agent = next((s["agent_info"] for s in summaries if s["agent_info"]["name"]),
|
|
361
|
+
summaries[0]["agent_info"])
|
|
362
|
+
with open(os.path.join(job_dir, "config.json"), "w", encoding="utf-8") as fh:
|
|
363
|
+
json.dump({
|
|
364
|
+
"job_name": job_name,
|
|
365
|
+
"harness": NAME, # provenance; the walker ignores unknown keys
|
|
366
|
+
"command": command or (
|
|
367
|
+
f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} "
|
|
368
|
+
f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"),
|
|
369
|
+
"agents": [{"name": agent["name"],
|
|
370
|
+
"model_name": agent["model_info"]["name"]}],
|
|
371
|
+
"tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries],
|
|
372
|
+
}, fh, indent=2)
|
|
373
|
+
return job_dir
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def convert(root: str, out_root: str, job_name: str = "",
|
|
377
|
+
command: str = "") -> list[str]:
|
|
378
|
+
"""Convert the documents at *root* — a directory of them or one file —
|
|
379
|
+
into one job. Every document is read before anything is written, so a
|
|
380
|
+
refused tree leaves no half-written job behind."""
|
|
381
|
+
docs = _read_all(root)
|
|
382
|
+
if not job_name:
|
|
383
|
+
base = os.path.basename(os.path.normpath(root))
|
|
384
|
+
if os.path.isfile(root):
|
|
385
|
+
base = _trial_name(root, root)
|
|
386
|
+
job_name = f"{NAME}__{base}"
|
|
387
|
+
return [_write_job(docs, root, out_root, job_name, command)]
|
nullius/importers/cli.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Turn a foreign harness's run tree into a job directory this app can walk.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.importers.cli <run tree> -o <job root>
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.importers.cli <run tree> -o <job root> --json
|
|
5
|
+
PYTHONPATH=app python3 -m nullius.importers.cli --list
|
|
6
|
+
|
|
7
|
+
The harness is detected from the directory's own shape; ``--harness NAME``
|
|
8
|
+
names one outright when detection would be ambiguous or a tree is unusual.
|
|
9
|
+
Each importer lives in ``nullius/importers/`` and registers by being
|
|
10
|
+
there, so this CLI never names one and gains nothing when one is added or
|
|
11
|
+
loses anything when one is deleted. With none registered ``--list`` says
|
|
12
|
+
``no importers registered`` rather than printing nothing, and every run tree
|
|
13
|
+
is refused as unrecognised.
|
|
14
|
+
|
|
15
|
+
Every job written is named on stdout by its id — the directory's basename,
|
|
16
|
+
which is what ``nullius.runs.cli list`` prints and ``/job/<id>`` opens — and
|
|
17
|
+
``--json`` gives the same as one document. The source tree is never edited;
|
|
18
|
+
the only writes land under ``-o``. The exit codes are the contract in
|
|
19
|
+
``docs/cli.md``, shared by every CLI here.
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import sys
|
|
27
|
+
|
|
28
|
+
from nullius import importers as harnesses
|
|
29
|
+
|
|
30
|
+
# Exit codes — docs/cli.md is the contract; the other two CLIs use the same
|
|
31
|
+
# numbers. 1 is left to Python itself: an uncaught exception is a bug.
|
|
32
|
+
EXIT_OK = 0
|
|
33
|
+
EXIT_NOT_FOUND = 3
|
|
34
|
+
EXIT_FAILED = 4
|
|
35
|
+
|
|
36
|
+
EPILOG = """\
|
|
37
|
+
examples:
|
|
38
|
+
PYTHONPATH=app python3 -m nullius.importers.cli /path/to/logs -o jobs_imported
|
|
39
|
+
PYTHONPATH=app python3 -m nullius.importers.cli /path/to/logs -o jobs_imported --json
|
|
40
|
+
PYTHONPATH=app python3 -m nullius.importers.cli --list
|
|
41
|
+
|
|
42
|
+
exit codes:
|
|
43
|
+
0 done
|
|
44
|
+
1 unexpected error (a traceback: report it)
|
|
45
|
+
2 usage: a bad flag, a missing argument, an unknown --harness
|
|
46
|
+
3 input not found: no such run tree, or no importer recognised it
|
|
47
|
+
4 the importer refused the tree: nothing under it could be converted
|
|
48
|
+
With --json, stdout is exactly one JSON document; everything else goes to stderr.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def main(argv=None) -> int:
|
|
53
|
+
ap = argparse.ArgumentParser(
|
|
54
|
+
prog="python3 -m nullius.importers.cli", description=__doc__.splitlines()[0],
|
|
55
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
56
|
+
ap.add_argument("run_dir", nargs="?", help="the harness's run tree")
|
|
57
|
+
ap.add_argument("-o", "--out", default="jobs_imported",
|
|
58
|
+
help="job root to write into (default: jobs_imported)")
|
|
59
|
+
ap.add_argument("--harness", default="",
|
|
60
|
+
help="name an importer instead of detecting one")
|
|
61
|
+
ap.add_argument("--job-name", default="", help="name for the job directory")
|
|
62
|
+
ap.add_argument("--command", default="",
|
|
63
|
+
help="the invocation that produced the run, shown verbatim "
|
|
64
|
+
"on the job page")
|
|
65
|
+
ap.add_argument("--list", action="store_true", help="list known harnesses")
|
|
66
|
+
ap.add_argument("--json", action="store_true",
|
|
67
|
+
help="one JSON document naming the harness and every job written")
|
|
68
|
+
args = ap.parse_args(argv)
|
|
69
|
+
|
|
70
|
+
known = harnesses.available()
|
|
71
|
+
if args.list:
|
|
72
|
+
if args.json:
|
|
73
|
+
json.dump({name: (module.__doc__ or "").strip().splitlines()[0]
|
|
74
|
+
for name, module in known.items()}, sys.stdout, indent=1)
|
|
75
|
+
sys.stdout.write("\n")
|
|
76
|
+
return EXIT_OK
|
|
77
|
+
# An empty registry is a state, not a crash - but printing nothing
|
|
78
|
+
# would read exactly like a CLI that imported and did no work.
|
|
79
|
+
if not known:
|
|
80
|
+
print("no importers registered")
|
|
81
|
+
for name, module in known.items():
|
|
82
|
+
summary = (module.__doc__ or "").strip().splitlines()[0]
|
|
83
|
+
print(f" {name:<10} {summary}")
|
|
84
|
+
return EXIT_OK
|
|
85
|
+
|
|
86
|
+
if not args.run_dir:
|
|
87
|
+
ap.error("run_dir is required (or pass --list)")
|
|
88
|
+
|
|
89
|
+
if args.harness:
|
|
90
|
+
module = harnesses.get(args.harness)
|
|
91
|
+
if module is None:
|
|
92
|
+
ap.error(f"unknown harness {args.harness!r}; known: {', '.join(known)}")
|
|
93
|
+
if not os.path.isdir(args.run_dir):
|
|
94
|
+
print(f"no such directory: {args.run_dir}", file=sys.stderr)
|
|
95
|
+
return EXIT_NOT_FOUND
|
|
96
|
+
if not args.harness:
|
|
97
|
+
module = harnesses.detect(args.run_dir)
|
|
98
|
+
if module is None:
|
|
99
|
+
print(f"no importer recognised {args.run_dir}; "
|
|
100
|
+
f"pass --harness (known: {', '.join(known) or 'none'})", file=sys.stderr)
|
|
101
|
+
return EXIT_NOT_FOUND
|
|
102
|
+
|
|
103
|
+
# Use the importer's tree boundaries so a parent can yield several jobs.
|
|
104
|
+
# Report refusal messages on stderr with EXIT_FAILED; preserve numeric exits.
|
|
105
|
+
try:
|
|
106
|
+
job_dirs = []
|
|
107
|
+
for tree in harnesses.trees(module, args.run_dir) or [args.run_dir]:
|
|
108
|
+
job_dirs += module.convert(tree, args.out, args.job_name if tree == args.run_dir else "", args.command)
|
|
109
|
+
except SystemExit as e:
|
|
110
|
+
if isinstance(e.code, int):
|
|
111
|
+
raise
|
|
112
|
+
print(e.code, file=sys.stderr)
|
|
113
|
+
return EXIT_FAILED
|
|
114
|
+
|
|
115
|
+
jobs = []
|
|
116
|
+
for job_dir in job_dirs:
|
|
117
|
+
trials = sum(
|
|
118
|
+
1 for name in os.listdir(job_dir)
|
|
119
|
+
if os.path.isfile(os.path.join(job_dir, name, "result.json"))
|
|
120
|
+
)
|
|
121
|
+
jobs.append({"job_id": os.path.basename(os.path.normpath(job_dir)),
|
|
122
|
+
"dir": job_dir, "trials": trials})
|
|
123
|
+
if args.json:
|
|
124
|
+
json.dump({"harness": module.NAME, "out": args.out, "jobs": jobs},
|
|
125
|
+
sys.stdout, ensure_ascii=False, indent=1)
|
|
126
|
+
sys.stdout.write("\n")
|
|
127
|
+
return EXIT_OK
|
|
128
|
+
for job in jobs:
|
|
129
|
+
print(f"{module.NAME}: wrote {job['trials']} trial(s) to {job['dir']} "
|
|
130
|
+
f"(job id {job['job_id']})")
|
|
131
|
+
return EXIT_OK
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__": # pragma: no cover - entry point
|
|
135
|
+
raise SystemExit(main())
|