nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,387 @@
1
+ """ATIF trajectories from any producer: standalone documents in, one job out.
2
+
3
+ PYTHONPATH=app python3 -m nullius.importers.cli trajectories/ -o jobs_imported
4
+ PYTHONPATH=app python3 -m nullius.importers.cli agent/trajectory.json -o jobs_imported
5
+
6
+ An ATIF document is the file Harbor's agents write as ``agent/trajectory.json``
7
+ and the reader already opens inside a trial (``nullius.traces.trajectory``).
8
+ On its own — dropped on the import picker, or a directory of them with no
9
+ ``result.json`` beside — nothing recognised it. This does, by content: a JSON
10
+ object whose ``schema_version`` reads ``ATIF-v1.x`` and which carries
11
+ ``steps``. The producer's name, the file's name and the directory it sits in
12
+ are not consulted, so a trajectory from claude-code, opencode or
13
+ any other adopter of the schema is read the same way, and two copies of one
14
+ document normalise identically wherever they were found.
15
+
16
+ **Supported schema versions**: ``ATIF-v1.0`` through ``ATIF-v1.7``
17
+ (``SUPPORTED_VERSIONS``) — the versions Nullius already parses inside a job,
18
+ as written by Harbor's agents (``final_metrics``, per-step
19
+ ``metrics``, the ``extra`` usage slot). A document of any other version is
20
+ refused by name rather than read as though it were one of these.
21
+
22
+ The app walks Harbor's shape, so this converts: one directory (or one file) becomes one job, each document
23
+ a trial directory holding the document *copied byte for byte* as
24
+ ``agent/trajectory.json``. Nothing is parsed twice — ``trajectory.load`` is
25
+ the one reader, here and on the trial page — and nothing is re-measured:
26
+
27
+ - **No benchmark outcome is invented.** A trajectory records what an agent
28
+ did, not whether it was right. The trial carries no ``verifier_result``,
29
+ no reward and no ``verifier/`` marker, so it reads as ``unknown`` — never
30
+ a score, never a solve rate — and stays out of every rate.
31
+ - **Usage and timing are the document's own, or absent.** ``agent_result``
32
+ copies ``final_metrics`` (prompt, completion, cached, cost) and is null
33
+ where the document is silent; the trial window is the span of the step
34
+ timestamps, and null when no step carries one. Nothing is zeroed to fill
35
+ a column: "unmeasured" and "zero" are different claims.
36
+ - **Provenance is the document's.** ``agent_info`` comes from the ATIF
37
+ ``agent`` block (name, version, model_name), falling back to the first
38
+ step's ``model_name`` for the model; the schema version, session id and
39
+ the source file's path and digest are kept on the trial record under
40
+ ``source_document``.
41
+ - **The recorded command is the import.** No harness ran here, so the line
42
+ the job page offers to copy is the one that reproduces this job
43
+ directory, not a ``harbor run`` that never happened.
44
+
45
+ A tree this claims but cannot read is refused before anything is written,
46
+ with a ``SystemExit`` whose message names the file (relative to the tree
47
+ given) and what is wrong — a document that is not JSON, one without
48
+ ``steps``, one of an unsupported version — and is redacted: a credential in
49
+ a bad file's name or content must not travel into the error. A ``.json``
50
+ that is well-formed but not ATIF at all (a ``trace.json``, a
51
+ ``spans.meta.json``) is simply not one of ours and is passed over.
52
+
53
+ What this does *not* claim: a Harbor job or trial (a ``result.json`` on the
54
+ walk). Those trajectories belong to the record around them, whose accounting
55
+ is the reader's and must not be shadowed by a second, thinner reading of the same
56
+ file.
57
+
58
+ Re-converting is idempotent: trial names are
59
+ deterministic (the document's path relative to the tree, a bare
60
+ ``trajectory.json`` named after its directory), every file is rewritten in
61
+ place, and a trial the tree no longer holds is removed.
62
+ """
63
+ from __future__ import annotations
64
+
65
+ import hashlib
66
+ import json
67
+ import os
68
+ import re
69
+ import shlex
70
+ import shutil
71
+ from datetime import timezone
72
+
73
+ from nullius.redact import redact
74
+ from nullius.traces.trajectory import parse_ts
75
+
76
+ NAME = "atif"
77
+ LABEL = "ATIF"
78
+
79
+ # The schema versions this reads — the ones `trajectory.py` is known to
80
+ # parse inside a Harbor job. Anchoring on the list is what keeps detect()
81
+ # from claiming a future shape and reading it as one of these.
82
+ SUPPORTED_VERSIONS = tuple(f"ATIF-v1.{minor}" for minor in range(8))
83
+
84
+ # What makes a JSON object an ATIF document at all, whatever its version:
85
+ # the schema tag. Shape (steps present, a list) is checked when read, so a
86
+ # partial document is refused by name rather than passed over as foreign.
87
+ _VERSION_RE = re.compile(r"^ATIF-v\d+\.\d+$")
88
+
89
+ # A file that is not JSON is only *ours* to refuse when it says it is an
90
+ # ATIF document; sniffed in the head of the file, since it cannot be parsed.
91
+ _MARKER = b'"ATIF-v'
92
+ _SNIFF_BYTES = 4096
93
+
94
+ # Files that mark a directory as part of a record another reader owns.
95
+ _OWNED_BY = ("result.json",)
96
+
97
+
98
+
99
+ def _is_atif(doc) -> bool:
100
+ return (isinstance(doc, dict)
101
+ and isinstance(doc.get("schema_version"), str)
102
+ and bool(_VERSION_RE.match(doc["schema_version"])))
103
+
104
+
105
+ def _mentions_atif(path: str) -> bool:
106
+ try:
107
+ with open(path, "rb") as fh:
108
+ return _MARKER in fh.read(_SNIFF_BYTES)
109
+ except OSError:
110
+ return False
111
+
112
+
113
+ def _read(path: str, shown: str) -> dict:
114
+ """The document at *path*, or a SystemExit naming what is wrong with it.
115
+
116
+ The message is what the CLI prints and the picker shows, so it is
117
+ redacted here, at construction, whichever branch built it.
118
+ """
119
+ try:
120
+ with open(path, encoding="utf-8") as fh:
121
+ doc = json.load(fh)
122
+ except OSError as e:
123
+ raise SystemExit(redact(f"{shown}: cannot be read ({e.strerror})")) from None
124
+ except ValueError as e:
125
+ raise SystemExit(redact(f"{shown}: not JSON ({e})")) from None
126
+ if not _is_atif(doc):
127
+ raise SystemExit(redact(f"{shown}: not an ATIF document"))
128
+ version = doc["schema_version"]
129
+ if version not in SUPPORTED_VERSIONS:
130
+ raise SystemExit(redact(
131
+ f"{shown}: unsupported ATIF schema_version {version!r} "
132
+ f"(supported: {SUPPORTED_VERSIONS[0]} through {SUPPORTED_VERSIONS[-1]})"))
133
+ if "steps" not in doc:
134
+ raise SystemExit(redact(f"{shown}: no steps (a partial document)"))
135
+ if not isinstance(doc["steps"], list):
136
+ raise SystemExit(redact(f"{shown}: steps is not a list"))
137
+ return doc
138
+
139
+
140
+ def _claims(path: str) -> bool:
141
+ """Is this file ours — an ATIF document, or a broken file that says it is?"""
142
+ try:
143
+ with open(path, encoding="utf-8") as fh:
144
+ doc = json.load(fh)
145
+ except OSError:
146
+ return False
147
+ except ValueError:
148
+ return _mentions_atif(path)
149
+ return _is_atif(doc)
150
+
151
+
152
+ def find_documents(root: str) -> list[str]:
153
+ """Every file under *root* this importer claims, sorted; *root* itself
154
+ when it is one file. Empty when a ``result.json`` is
155
+ seen on the walk — that tree is a record another reader owns."""
156
+ if os.path.isfile(root):
157
+ return [root] if _claims(root) else []
158
+ found = []
159
+ for dirpath, dirnames, filenames in os.walk(root):
160
+ dirnames.sort()
161
+ if any(name in filenames for name in _OWNED_BY):
162
+ return []
163
+ for name in sorted(filenames):
164
+ path = os.path.join(dirpath, name)
165
+ if name.endswith(".json") and _claims(path):
166
+ found.append(path)
167
+ return found
168
+
169
+
170
+ def detect(path: str) -> bool:
171
+ """Does *path* hold standalone ATIF documents (or is it one)?"""
172
+ return bool(find_documents(path))
173
+
174
+
175
+ def check(path: str) -> None:
176
+ """Read every document the way `convert` would, writing nothing; raises
177
+ ``SystemExit`` (redacted) for the first thing `convert` would refuse."""
178
+ _read_all(path)
179
+
180
+
181
+ def trees(root: str) -> list[str]:
182
+ """One directory, one job, whatever its depth — the CLI's reading of
183
+ *root*, shared by the picker. The documents found under it are
184
+ that job's trials, named by their place in the tree (`_trial_name`), so
185
+ a nested directory of trajectories is one job of several trials rather
186
+ than a job per child."""
187
+ return [root] if detect(root) else []
188
+
189
+
190
+ def _shown(path: str, root: str) -> str:
191
+ """How a file is named in a message: relative to the tree given."""
192
+ if os.path.isfile(root):
193
+ return os.path.basename(path)
194
+ return os.path.relpath(path, root)
195
+
196
+
197
+ def _read_all(root: str) -> list[tuple[str, dict]]:
198
+ paths = find_documents(root)
199
+ if not paths:
200
+ raise SystemExit(redact(
201
+ f"no ATIF trajectories (schema_version {SUPPORTED_VERSIONS[0]} through "
202
+ f"{SUPPORTED_VERSIONS[-1]}) found in {root}"))
203
+ return [(path, _read(path, _shown(path, root))) for path in paths]
204
+
205
+
206
+
207
+ def _trial_name(path: str, root: str) -> str:
208
+ """The document's place in the tree, flattened for a directory name.
209
+
210
+ A bare ``trajectory.json`` is named after the directory holding it, so
211
+ ``claude-code/trajectory.json`` and ``claude-code.json`` are the same
212
+ trial under either layout. A single file, or a ``trajectory.json`` at
213
+ the root itself, takes the file's own stem.
214
+ """
215
+ if os.path.isfile(root):
216
+ rel = os.path.basename(path)
217
+ else:
218
+ rel = os.path.relpath(path, root)
219
+ stem = rel[:-len(".json")] if rel.endswith(".json") else rel
220
+ parts = [p for p in stem.split(os.sep) if p and p != "."]
221
+ if len(parts) > 1 and parts[-1] == "trajectory":
222
+ parts.pop()
223
+ return "__".join(parts) or "trajectory"
224
+
225
+
226
+ def _instant(stamp: str) -> float:
227
+ """A stamp as seconds since the epoch, a naive one read as UTC: what the
228
+ job's first start and last end are chosen by."""
229
+ when = parse_ts(stamp)
230
+ if when is None:
231
+ return float("inf")
232
+ return (when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when).timestamp()
233
+
234
+
235
+ def _window(doc: dict) -> tuple[str | None, str | None]:
236
+ """The first and last step timestamps that parse, in step order and as
237
+ recorded; None when no step carries a clock — the trial then has no
238
+ duration, not a zero one. Step order rather than min/max: the document's
239
+ ordering is preserved, and two clocks of different kinds (one naive, one
240
+ zoned) are never compared."""
241
+ stamps = [s.get("timestamp") for s in doc["steps"]
242
+ if isinstance(s, dict) and parse_ts(s.get("timestamp")) is not None]
243
+ if not stamps:
244
+ return None, None
245
+ return stamps[0], stamps[-1]
246
+
247
+
248
+ def _agent_info(doc: dict) -> dict:
249
+ agent = doc.get("agent") if isinstance(doc.get("agent"), dict) else {}
250
+ model = agent.get("model_name") or next(
251
+ (s.get("model_name") for s in doc["steps"]
252
+ if isinstance(s, dict) and s.get("model_name")), "")
253
+ return {
254
+ "name": str(agent.get("name") or ""),
255
+ "version": str(agent.get("version") or ""),
256
+ "model_info": {"name": str(model or "")},
257
+ }
258
+
259
+
260
+ def _usage(doc: dict) -> dict:
261
+ """``final_metrics`` in the fields the reader sums — null where absent."""
262
+ metrics = doc.get("final_metrics") if isinstance(doc.get("final_metrics"), dict) else {}
263
+
264
+ def number(key):
265
+ value = metrics.get(key)
266
+ return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
267
+
268
+ return {
269
+ "n_input_tokens": number("total_prompt_tokens"),
270
+ "n_output_tokens": number("total_completion_tokens"),
271
+ "n_cache_tokens": number("total_cached_tokens"),
272
+ "cost_usd": number("total_cost_usd"),
273
+ }
274
+
275
+
276
+ def _sha256(path: str) -> str:
277
+ digest = hashlib.sha256()
278
+ with open(path, "rb") as fh:
279
+ for chunk in iter(lambda: fh.read(1 << 16), b""):
280
+ digest.update(chunk)
281
+ return digest.hexdigest()
282
+
283
+
284
+ # ── one document = one trial ────────────────────────────────────────────────
285
+
286
+ def convert_document(path: str, doc: dict, trial: str, job_dir: str) -> dict:
287
+ """Write one document as a Harbor-shaped trial directory. Returns a summary."""
288
+ trial_dir = os.path.join(job_dir, trial)
289
+ agent_dir = os.path.join(trial_dir, "agent")
290
+ os.makedirs(agent_dir, exist_ok=True)
291
+
292
+ started, finished = _window(doc)
293
+ result = {
294
+ "trial_name": trial,
295
+ "task_name": f"{NAME}/{trial}",
296
+ # The harness, not a dataset: the benchmark readers stay unengaged,
297
+ # and with no report and no reward the outcome is `unknown`.
298
+ "source": NAME,
299
+ "agent_info": _agent_info(doc),
300
+ "exception_info": {},
301
+ "verifier_result": {},
302
+ "started_at": started,
303
+ "finished_at": finished,
304
+ "environment_setup": None,
305
+ "agent_setup": None,
306
+ # The document is the agent's own window; what surrounded it (image
307
+ # pull, setup, a verifier) was never recorded here.
308
+ "agent_execution": (
309
+ {"started_at": started, "finished_at": finished} if started else None),
310
+ "verifier": None,
311
+ "agent_result": _usage(doc),
312
+ "source_document": {
313
+ "path": os.path.abspath(path),
314
+ "sha256": _sha256(path),
315
+ "schema_version": doc["schema_version"],
316
+ "session_id": doc.get("session_id"),
317
+ },
318
+ }
319
+ with open(os.path.join(trial_dir, "result.json"), "w", encoding="utf-8") as fh:
320
+ json.dump(result, fh, indent=2)
321
+
322
+ # Byte for byte: the trial page reads what the producer wrote, and a
323
+ # digest of the copy is a digest of the source.
324
+ shutil.copyfile(path, os.path.join(agent_dir, "trajectory.json"))
325
+
326
+ return {"trial_dir": trial, "started_at": started, "finished_at": finished,
327
+ "agent_info": result["agent_info"]}
328
+
329
+
330
+
331
+ def _write_job(docs: list, root: str, out_root: str, job_name: str, command: str) -> str:
332
+ """The read documents, as one job directory of one trial each."""
333
+ job_dir = os.path.join(out_root, job_name)
334
+ os.makedirs(job_dir, exist_ok=True)
335
+
336
+ summaries = [convert_document(path, doc, _trial_name(path, root), job_dir)
337
+ for path, doc in docs]
338
+ # Derived output this importer wholly owns: a trial the tree no longer
339
+ # holds is stale and would keep being counted.
340
+ produced = {s["trial_dir"] for s in summaries}
341
+ for name in os.listdir(job_dir):
342
+ sub = os.path.join(job_dir, name)
343
+ if os.path.isdir(sub) and name not in produced:
344
+ shutil.rmtree(sub)
345
+
346
+ # ordered as instants, not as text: one stamp with a zone and one without sort apart as strings
347
+ starts = sorted((s["started_at"] for s in summaries if s["started_at"]), key=_instant)
348
+ ends = sorted((s["finished_at"] for s in summaries if s["finished_at"]), key=_instant)
349
+ with open(os.path.join(job_dir, "result.json"), "w", encoding="utf-8") as fh:
350
+ json.dump({
351
+ "started_at": starts[0] if starts else None,
352
+ "finished_at": ends[-1] if ends else None,
353
+ "n_total_trials": len(summaries),
354
+ "stats": {"n_errored_trials": 0, "n_running_trials": 0,
355
+ "n_pending_trials": 0, "n_retries": 0},
356
+ }, fh, indent=2)
357
+
358
+ # The first document's producer names the job; a tree that mixes
359
+ # producers still shows each trial's own under agent_info.
360
+ agent = next((s["agent_info"] for s in summaries if s["agent_info"]["name"]),
361
+ summaries[0]["agent_info"])
362
+ with open(os.path.join(job_dir, "config.json"), "w", encoding="utf-8") as fh:
363
+ json.dump({
364
+ "job_name": job_name,
365
+ "harness": NAME, # provenance; the walker ignores unknown keys
366
+ "command": command or (
367
+ f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} "
368
+ f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"),
369
+ "agents": [{"name": agent["name"],
370
+ "model_name": agent["model_info"]["name"]}],
371
+ "tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries],
372
+ }, fh, indent=2)
373
+ return job_dir
374
+
375
+
376
+ def convert(root: str, out_root: str, job_name: str = "",
377
+ command: str = "") -> list[str]:
378
+ """Convert the documents at *root* — a directory of them or one file —
379
+ into one job. Every document is read before anything is written, so a
380
+ refused tree leaves no half-written job behind."""
381
+ docs = _read_all(root)
382
+ if not job_name:
383
+ base = os.path.basename(os.path.normpath(root))
384
+ if os.path.isfile(root):
385
+ base = _trial_name(root, root)
386
+ job_name = f"{NAME}__{base}"
387
+ return [_write_job(docs, root, out_root, job_name, command)]
@@ -0,0 +1,135 @@
1
+ """Turn a foreign harness's run tree into a job directory this app can walk.
2
+
3
+ PYTHONPATH=app python3 -m nullius.importers.cli <run tree> -o <job root>
4
+ PYTHONPATH=app python3 -m nullius.importers.cli <run tree> -o <job root> --json
5
+ PYTHONPATH=app python3 -m nullius.importers.cli --list
6
+
7
+ The harness is detected from the directory's own shape; ``--harness NAME``
8
+ names one outright when detection would be ambiguous or a tree is unusual.
9
+ Each importer lives in ``nullius/importers/`` and registers by being
10
+ there, so this CLI never names one and gains nothing when one is added or
11
+ loses anything when one is deleted. With none registered ``--list`` says
12
+ ``no importers registered`` rather than printing nothing, and every run tree
13
+ is refused as unrecognised.
14
+
15
+ Every job written is named on stdout by its id — the directory's basename,
16
+ which is what ``nullius.runs.cli list`` prints and ``/job/<id>`` opens — and
17
+ ``--json`` gives the same as one document. The source tree is never edited;
18
+ the only writes land under ``-o``. The exit codes are the contract in
19
+ ``docs/cli.md``, shared by every CLI here.
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import argparse
24
+ import json
25
+ import os
26
+ import sys
27
+
28
+ from nullius import importers as harnesses
29
+
30
+ # Exit codes — docs/cli.md is the contract; the other two CLIs use the same
31
+ # numbers. 1 is left to Python itself: an uncaught exception is a bug.
32
+ EXIT_OK = 0
33
+ EXIT_NOT_FOUND = 3
34
+ EXIT_FAILED = 4
35
+
36
+ EPILOG = """\
37
+ examples:
38
+ PYTHONPATH=app python3 -m nullius.importers.cli /path/to/logs -o jobs_imported
39
+ PYTHONPATH=app python3 -m nullius.importers.cli /path/to/logs -o jobs_imported --json
40
+ PYTHONPATH=app python3 -m nullius.importers.cli --list
41
+
42
+ exit codes:
43
+ 0 done
44
+ 1 unexpected error (a traceback: report it)
45
+ 2 usage: a bad flag, a missing argument, an unknown --harness
46
+ 3 input not found: no such run tree, or no importer recognised it
47
+ 4 the importer refused the tree: nothing under it could be converted
48
+ With --json, stdout is exactly one JSON document; everything else goes to stderr.
49
+ """
50
+
51
+
52
+ def main(argv=None) -> int:
53
+ ap = argparse.ArgumentParser(
54
+ prog="python3 -m nullius.importers.cli", description=__doc__.splitlines()[0],
55
+ epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
56
+ ap.add_argument("run_dir", nargs="?", help="the harness's run tree")
57
+ ap.add_argument("-o", "--out", default="jobs_imported",
58
+ help="job root to write into (default: jobs_imported)")
59
+ ap.add_argument("--harness", default="",
60
+ help="name an importer instead of detecting one")
61
+ ap.add_argument("--job-name", default="", help="name for the job directory")
62
+ ap.add_argument("--command", default="",
63
+ help="the invocation that produced the run, shown verbatim "
64
+ "on the job page")
65
+ ap.add_argument("--list", action="store_true", help="list known harnesses")
66
+ ap.add_argument("--json", action="store_true",
67
+ help="one JSON document naming the harness and every job written")
68
+ args = ap.parse_args(argv)
69
+
70
+ known = harnesses.available()
71
+ if args.list:
72
+ if args.json:
73
+ json.dump({name: (module.__doc__ or "").strip().splitlines()[0]
74
+ for name, module in known.items()}, sys.stdout, indent=1)
75
+ sys.stdout.write("\n")
76
+ return EXIT_OK
77
+ # An empty registry is a state, not a crash - but printing nothing
78
+ # would read exactly like a CLI that imported and did no work.
79
+ if not known:
80
+ print("no importers registered")
81
+ for name, module in known.items():
82
+ summary = (module.__doc__ or "").strip().splitlines()[0]
83
+ print(f" {name:<10} {summary}")
84
+ return EXIT_OK
85
+
86
+ if not args.run_dir:
87
+ ap.error("run_dir is required (or pass --list)")
88
+
89
+ if args.harness:
90
+ module = harnesses.get(args.harness)
91
+ if module is None:
92
+ ap.error(f"unknown harness {args.harness!r}; known: {', '.join(known)}")
93
+ if not os.path.isdir(args.run_dir):
94
+ print(f"no such directory: {args.run_dir}", file=sys.stderr)
95
+ return EXIT_NOT_FOUND
96
+ if not args.harness:
97
+ module = harnesses.detect(args.run_dir)
98
+ if module is None:
99
+ print(f"no importer recognised {args.run_dir}; "
100
+ f"pass --harness (known: {', '.join(known) or 'none'})", file=sys.stderr)
101
+ return EXIT_NOT_FOUND
102
+
103
+ # Use the importer's tree boundaries so a parent can yield several jobs.
104
+ # Report refusal messages on stderr with EXIT_FAILED; preserve numeric exits.
105
+ try:
106
+ job_dirs = []
107
+ for tree in harnesses.trees(module, args.run_dir) or [args.run_dir]:
108
+ job_dirs += module.convert(tree, args.out, args.job_name if tree == args.run_dir else "", args.command)
109
+ except SystemExit as e:
110
+ if isinstance(e.code, int):
111
+ raise
112
+ print(e.code, file=sys.stderr)
113
+ return EXIT_FAILED
114
+
115
+ jobs = []
116
+ for job_dir in job_dirs:
117
+ trials = sum(
118
+ 1 for name in os.listdir(job_dir)
119
+ if os.path.isfile(os.path.join(job_dir, name, "result.json"))
120
+ )
121
+ jobs.append({"job_id": os.path.basename(os.path.normpath(job_dir)),
122
+ "dir": job_dir, "trials": trials})
123
+ if args.json:
124
+ json.dump({"harness": module.NAME, "out": args.out, "jobs": jobs},
125
+ sys.stdout, ensure_ascii=False, indent=1)
126
+ sys.stdout.write("\n")
127
+ return EXIT_OK
128
+ for job in jobs:
129
+ print(f"{module.NAME}: wrote {job['trials']} trial(s) to {job['dir']} "
130
+ f"(job id {job['job_id']})")
131
+ return EXIT_OK
132
+
133
+
134
+ if __name__ == "__main__": # pragma: no cover - entry point
135
+ raise SystemExit(main())