nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,274 @@
1
+ """OpenTelemetry span exports: a directory of OTLP/JSON files in, one job out.
2
+
3
+ PYTHONPATH=app python3 -m nullius.importers.cli traces-export/ -o jobs_imported
4
+
5
+ A production trace export is not a benchmark run — no task, no verifier, no
6
+ reward — but it is spans with clocks, which is what the trial page's waterfall
7
+ and ``/charts`` draw. So this lays the export out as the one shape the app
8
+ walks: a job directory holding **one trial per trace id**, each with the
9
+ ``harbor-trace-v1`` document ``traces/otel.py`` builds under ``agent/``. The
10
+ format, the conventions read and the field mapping are that module's; this one
11
+ decides what a trial *is* and what its ``result.json`` may honestly say.
12
+
13
+ What it says is little, on purpose:
14
+
15
+ - **The outcome is unknown, and no reward exists.** A span in ERROR marks that
16
+ span in the trace; it is not an exception on the trial, because a failed
17
+ call is not a failed task and the app's outcome taxonomy is about tasks.
18
+ ``exception_info`` and ``verifier_result`` are therefore empty, which
19
+ ``classify`` reads as *unknown* — not resolved, not failed.
20
+ - **Tokens are the trace's own totals** — the sum over its LLM spans — written
21
+ only when at least one span reported usage; a trace with no usage attributes
22
+ records no ``agent_result`` at all rather than three zeros.
23
+ - **The model is the first LLM span's**, the agent the resource's
24
+ ``service.name``. An export that names neither shows neither.
25
+ - **The clock is the trace's window**, first span start to last span end, as
26
+ both the trial's ``started_at``/``finished_at`` and its ``agent_execution``
27
+ phase — the only phase there is evidence for.
28
+
29
+ Recognition is by content: a ``.json`` file whose top level is an
30
+ ``ExportTraceServiceRequest`` (``resourceSpans``), or a JSON-lines file of them
31
+ — under the directory at any depth. What one job is under a root the app opened is this module's answer
32
+ (``trees``): an export directory is one job, and a parent that holds no
33
+ export of its own but several export directories is one job per directory —
34
+ each is one producer's export (the trial's agent is its ``service.name``),
35
+ and pooling them would put several agents under one heading. The CLI
36
+ converts the directory it is given as one job; a parent given to it is one
37
+ pooled job, which is the one place the two still read a tree differently.
38
+ Nothing Harbor writes carries ``resourceSpans``, and an ATIF
39
+ trajectory carries ``schema_version`` instead, so nothing here claims their
40
+ directories.
41
+
42
+ A file the reader refuses — bad JSON, no ``resourceSpans``, an unsupported
43
+ schema URL — is reported with its redacted message and skipped; only when
44
+ *nothing* converts is that fatal. The source directory is never written to:
45
+ everything lands under the output root, and a re-convert rewrites the same
46
+ names in place (a trace the export no longer holds is removed), so refreshing
47
+ a workspace updates rather than duplicates.
48
+ """
49
+ from __future__ import annotations
50
+
51
+ import json
52
+ import os
53
+ import re
54
+ import shutil
55
+ from datetime import datetime, timezone
56
+
57
+ from nullius.redact import redact
58
+ from nullius.traces import otel as reader
59
+ from nullius.traces import trace
60
+
61
+ NAME = "otel"
62
+ LABEL = "OpenTelemetry"
63
+
64
+ # Directory-name characters for a trial: the root span's name can be anything
65
+ # a producer chose, and a slash in it would nest the trial.
66
+ _UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
67
+
68
+
69
+
70
+ def find_exports(root: str) -> list[str]:
71
+ """Every OTLP/JSON file under *root*, sorted. A cheap head check first,
72
+ then the real read: the walk runs on every workspace open, over roots
73
+ that may hold thousands of Harbor files that are not this."""
74
+ out = []
75
+ for dirpath, _dirnames, filenames in os.walk(root):
76
+ for name in filenames:
77
+ path = os.path.join(dirpath, name)
78
+ if not (name.endswith(".json") or name.endswith(".jsonl")):
79
+ continue
80
+ if not reader.looks_like_export(path):
81
+ continue
82
+ try:
83
+ if reader.read_file(path):
84
+ out.append(path)
85
+ except reader.ReadError:
86
+ continue
87
+ return sorted(out)
88
+
89
+
90
+ def detect(path: str) -> bool:
91
+ """Does *path* hold at least one readable OTLP/JSON trace export?"""
92
+ return os.path.isdir(path) and bool(find_exports(path))
93
+
94
+
95
+ def trees(root: str) -> list[str]:
96
+ """The directories under *root* that are one job each.
97
+
98
+ *root* itself when an export file sits directly in it — an export
99
+ directory is one job however its files are arranged below. Otherwise
100
+ each child directory holding an export, one job per producer's export
101
+ rather than one pooled job of several agents. Nothing when there is no
102
+ export under *root* at all.
103
+ """
104
+ exports = find_exports(root) if os.path.isdir(root) else []
105
+ if not exports:
106
+ return []
107
+ if any(os.path.dirname(path) == root for path in exports):
108
+ return [root]
109
+ return [os.path.join(root, name) for name in sorted(os.listdir(root))
110
+ if os.path.isdir(os.path.join(root, name)) and detect(os.path.join(root, name))]
111
+
112
+
113
+ def refused(root: str) -> list[str]:
114
+ """(path, redacted reason) for every candidate file the reader refuses —
115
+ what the CLI reports beside the job it wrote."""
116
+ out = []
117
+ for dirpath, _dirnames, filenames in os.walk(root):
118
+ for name in sorted(filenames):
119
+ path = os.path.join(dirpath, name)
120
+ if not (name.endswith(".json") or name.endswith(".jsonl")):
121
+ continue
122
+ if not reader.looks_like_export(path):
123
+ continue
124
+ try:
125
+ reader.read_file(path)
126
+ except reader.ReadError as exc:
127
+ # The reader names the file by basename; this names it by its
128
+ # path under the root, once, in place of that.
129
+ reason = str(exc)
130
+ prefix = os.path.basename(path) + ": "
131
+ if reason.startswith(prefix):
132
+ reason = reason[len(prefix):]
133
+ out.append(f"{os.path.relpath(path, root)}: {reason}")
134
+ return out
135
+
136
+
137
+
138
+ def _iso(ns: int) -> str:
139
+ return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat().replace("+00:00", "Z")
140
+
141
+
142
+ def _dump(path: str, data) -> None:
143
+ with open(path, "w", encoding="utf-8") as fh:
144
+ json.dump(data, fh, ensure_ascii=False, indent=1)
145
+ fh.write("\n")
146
+
147
+
148
+ def _window(document: dict) -> tuple:
149
+ """(start_ns, end_ns) over every span, end None when no span ended."""
150
+ starts, ends = [], []
151
+ for raw in trace.raw_spans_of(document.get("spans") or []):
152
+ starts.append(raw["start_ns"])
153
+ if raw.get("end_ns") is not None:
154
+ ends.append(raw["end_ns"])
155
+ return (min(starts) if starts else None, max(ends) if ends else None)
156
+
157
+
158
+ def trial_name_of(document: dict) -> str:
159
+ """``<root span name>__<7 hex of the trace id>`` — readable, and unique
160
+ per trace the way a Harbor trial name is unique per attempt."""
161
+ tid = str(document.get("session_id") or "")
162
+ stem = _UNSAFE.sub("_", str(document.get("trial_name") or "")).strip("._-") or "trace"
163
+ return f"{stem[:80]}__{tid[:7] or 'unknown'}"
164
+
165
+
166
+ def write_trial(document: dict, job_dir: str) -> dict:
167
+ """One trace document as a trial directory. Returns a summary."""
168
+ trial = trial_name_of(document)
169
+ trial_dir = os.path.join(job_dir, trial)
170
+ agent_dir = os.path.join(trial_dir, "agent")
171
+ os.makedirs(agent_dir, exist_ok=True)
172
+
173
+ start_ns, end_ns = _window(document)
174
+ started = _iso(start_ns) if start_ns else None
175
+ finished = _iso(end_ns) if end_ns else None
176
+ tokens = (document.get("totals") or {}).get("tokens") or {}
177
+ model = reader.model_of(document)
178
+ agent = document.get("agent") or {}
179
+
180
+ result = {
181
+ "trial_name": trial,
182
+ "task_name": f"{NAME}/{document.get('trial_name') or trial}",
183
+ "source": NAME,
184
+ "agent_info": {
185
+ "name": agent.get("name") or "",
186
+ "version": agent.get("version") or "",
187
+ "model_info": {"name": model},
188
+ },
189
+ # Deliberately empty: a span's status is not a task's outcome, and no
190
+ # verifier ran. The walker reads this pair as `unknown`.
191
+ "exception_info": {},
192
+ "verifier_result": {},
193
+ "started_at": started,
194
+ "finished_at": finished,
195
+ "environment_setup": None,
196
+ "agent_setup": None,
197
+ "agent_execution": (
198
+ {"started_at": started, "finished_at": finished} if started and finished else None),
199
+ "verifier": None,
200
+ "config": {"environment": {"type": ""}},
201
+ }
202
+ if tokens:
203
+ result["agent_result"] = {
204
+ "n_input_tokens": tokens.get("input_tokens", 0),
205
+ "n_output_tokens": tokens.get("output_tokens", 0),
206
+ "n_cache_tokens": tokens.get("cache_read_tokens", 0),
207
+ "cost_usd": None,
208
+ }
209
+ _dump(os.path.join(trial_dir, "result.json"), result)
210
+ _dump(os.path.join(agent_dir, "trace.json"), document)
211
+ return {"trial_dir": trial, "started_at": started, "finished_at": finished,
212
+ "model": model, "agent": agent.get("name") or ""}
213
+
214
+
215
+ def _write_job(documents: list, out_root: str, job_name: str, command: str,
216
+ root: str, skipped: list) -> str:
217
+ job_dir = os.path.join(out_root, job_name)
218
+ os.makedirs(job_dir, exist_ok=True)
219
+ summaries = [write_trial(doc, job_dir) for doc in documents]
220
+
221
+ # Derived output this importer wholly owns: a trial the export no longer
222
+ # produces is stale and would keep being counted.
223
+ produced = {s["trial_dir"] for s in summaries}
224
+ for name in os.listdir(job_dir):
225
+ sub = os.path.join(job_dir, name)
226
+ if os.path.isdir(sub) and name not in produced:
227
+ shutil.rmtree(sub)
228
+
229
+ starts = sorted(s["started_at"] for s in summaries if s["started_at"])
230
+ ends = sorted(s["finished_at"] for s in summaries if s["finished_at"])
231
+ _dump(os.path.join(job_dir, "result.json"), {
232
+ "started_at": starts[0] if starts else None,
233
+ "finished_at": ends[-1] if ends else None,
234
+ "n_total_trials": len(summaries),
235
+ "stats": {"n_errored_trials": 0, "n_running_trials": 0,
236
+ "n_pending_trials": 0, "n_retries": 0},
237
+ })
238
+ agents = sorted({s["agent"] for s in summaries if s["agent"]}) or [""]
239
+ models = sorted({s["model"] for s in summaries if s["model"]})
240
+ _dump(os.path.join(job_dir, "config.json"), {
241
+ "job_name": job_name,
242
+ "harness": NAME,
243
+ "command": command,
244
+ "agents": [{"name": agents[0], "model_name": models[0] if len(models) == 1 else "",
245
+ "kwargs": {}}],
246
+ "datasets": [{"name": NAME, "path": root, "n_tasks": len(summaries)}],
247
+ "import": {"format": reader.FORMAT, "conventions": reader.CONVENTIONS,
248
+ "skipped": skipped},
249
+ })
250
+ return job_dir
251
+
252
+
253
+ def convert(root: str, out_root: str, job_name: str = "",
254
+ command: str = "") -> list[str]:
255
+ """Convert every OTLP/JSON export under *root* into one job directory.
256
+
257
+ Traces are grouped across files, one trial each. Files the reader refuses
258
+ are listed in the job's ``config.json`` under ``import.skipped`` with their
259
+ redacted reasons; when nothing reads, that list is the error.
260
+ """
261
+ paths = find_exports(root)
262
+ skipped = refused(root)
263
+ if not paths:
264
+ raise SystemExit(redact(
265
+ f"no OTLP/JSON trace exports (resourceSpans) found in {root}"
266
+ + ("; refused: " + "; ".join(skipped) if skipped else "")))
267
+ try:
268
+ documents = reader.read(paths)
269
+ except reader.ReadError as exc: # a file changed under us since find_exports
270
+ raise SystemExit(str(exc)) from None
271
+ if not documents:
272
+ raise SystemExit(redact(f"the OTLP/JSON exports in {root} carry no spans"))
273
+ name = job_name or f"{NAME}__{os.path.basename(os.path.abspath(root)) or 'export'}"
274
+ return [_write_job(documents, out_root, name, command, root, skipped)]
@@ -0,0 +1,14 @@
1
+ """Judgement: structured, confidence-scored answers about a recorded trial
2
+ from a judge model (TypeSafe.ai System One), shown beside the deterministic
3
+ evidence and never in its place.
4
+
5
+ The contract: the question
6
+ vocabulary (``questions.py``, one id per question, versioned as a set), the
7
+ state each question reads (``state.py``, built from the redacted views only,
8
+ truncated head-and-tail with a marker), the record shape (``client.py``),
9
+ and where a judgement lives - the workspace store, keyed by trial and
10
+ question version (``store.py``), never under the job's own tree. A surface
11
+ asks through ``batch.py``: the questions grouped by state, one call per
12
+ state, the records merged and stored - so no page groups or merges on its
13
+ own.
14
+ """
nullius/judge/batch.py ADDED
@@ -0,0 +1,122 @@
1
+ """Asking a set of questions over the states they read, and keeping the
2
+ answers: the seam every surface uses, so a page never groups questions by
3
+ state or merges records on its own.
4
+
5
+ ``states_for(ids, build)`` groups the ids by the state kind each reads and
6
+ builds each state once; ``ask_states`` makes one call per state and merges
7
+ the records (``client.merge``); ``ask_and_store`` does that and saves the
8
+ result under the trial's key in the workspace store, merged over what was
9
+ there. A new surface registers its group (``questions.register``) and, if
10
+ its questions read a state of their own, that state's builder
11
+ (``state.register``), then calls these with its own key.
12
+
13
+ Every call also carries the ``hygiene`` group - does the state address the
14
+ judge, does it still show a credential - because state is untrusted text
15
+ from a job directory and output tokens are free. Their answers never land
16
+ in ``answers``: they go under ``hygiene[kind]`` in the record, one reading
17
+ per state kind, so a surface shows the flag beside every answer drawn from
18
+ that state and no answer changes because of it."""
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ from typing import Callable, Optional
23
+
24
+ from . import client, questions as Q, store
25
+
26
+
27
+ def with_hygiene(ids) -> list:
28
+ """The ids a call sends: the surface's own, then the hygiene group."""
29
+ return list(ids) + [q for q in Q.hygiene_ids() if q not in ids]
30
+
31
+
32
+ def split_hygiene(rec: dict, kind: str) -> dict:
33
+ """Move the hygiene answers out of `rec["answers"]` into
34
+ `rec["hygiene"][kind]` - `{id: probability}` - so the surface's answers
35
+ read as before and the reading is kept per state kind."""
36
+ readings = {}
37
+ for q in Q.hygiene_ids():
38
+ a = rec["answers"].pop(q, None)
39
+ if a and isinstance(a.get("noul"), (int, float)):
40
+ readings[q] = a["noul"]
41
+ if readings:
42
+ rec.setdefault("hygiene", {})[kind] = readings
43
+ return rec
44
+
45
+
46
+ def states_for(ids, build: Callable) -> dict:
47
+ """{kind: (ids, text, truncated)}: the questions grouped by the state
48
+ they read, each state built once by `build(kind) -> (text, truncated)`."""
49
+ groups: dict = {}
50
+ for q in ids:
51
+ groups.setdefault(Q.state_of(q), []).append(q)
52
+ out = {}
53
+ for kind, qs in groups.items():
54
+ text, truncated = build(kind)
55
+ out[kind] = (qs, text, truncated)
56
+ return out
57
+
58
+
59
+ def preflight(states: dict, calibration: Optional[dict] = None, budget: int = client.BUDGET_TOKENS) -> dict:
60
+ """Refuse locally a call the judge would refuse: each state's text plus
61
+ its questions' JSON, estimated at the upper edge of the band, against
62
+ the request budget. Raises JudgeError(413) naming the state kind and
63
+ roughly how far over it is; returns {kind: tokens_high} otherwise.
64
+
65
+ state.LIMIT is 60,000 characters - about 20,000 tokens at the measured
66
+ 3 per token, 24,000 at the band's edge of 2.5 - so a built-in state fits on the default
67
+ of thumb. The check matters when a calibration says the judge counts
68
+ fewer characters per token than that, and for a state kind a later
69
+ group registers with a limit of its own."""
70
+ out = {}
71
+ for kind, (qs, text, _) in states.items():
72
+ if not qs:
73
+ continue
74
+ chars = len(text) + len(json.dumps(Q.request_questions(with_hygiene(qs))))
75
+ est = client.estimate(chars, calls=1, calibration=calibration)
76
+ out[kind] = est["tokens_high"]
77
+ if est["tokens_high"] > budget:
78
+ over = est["tokens_high"] - budget
79
+ raise client.JudgeError(413, f"the `{kind}` state is too big for one judge call: up to {est['tokens_high']:,} tokens "
80
+ f"({chars:,} characters, {est['basis']}) against a budget of {budget:,}; "
81
+ f"about {over:,} tokens - roughly {int(over * est['chars_per_token']):,} characters - over. Nothing was sent.")
82
+ return out
83
+
84
+
85
+ def ask_states(states: dict, key: str, post: Optional[Callable] = None, calibration: Optional[dict] = None,
86
+ budget: int = client.BUDGET_TOKENS) -> Optional[dict]:
87
+ """One judge call per state, the records merged into one; None when
88
+ there was nothing to ask. `preflight` runs first, so a batch the judge
89
+ would refuse is refused here before any call - and before any charge.
90
+ Each call carries the hygiene group; its readings are kept per state
91
+ kind under `hygiene`, apart from `answers`."""
92
+ preflight(states, calibration=calibration, budget=budget)
93
+ records = []
94
+ for kind, (qs, text, truncated) in states.items():
95
+ if qs:
96
+ records.append(split_hygiene(client.ask(text, with_hygiene(qs), key, truncated=truncated, post=post, kind=kind), kind))
97
+ merged = client.merge(records)
98
+ if merged is not None:
99
+ hygiene = {}
100
+ for rec in records:
101
+ hygiene.update(rec.get("hygiene") or {})
102
+ merged["hygiene"] = hygiene
103
+ return merged
104
+
105
+
106
+ def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, post: Optional[Callable] = None,
107
+ calibration: Optional[dict] = None) -> Optional[dict]:
108
+ """`ask_states`, then the merged record saved under the trial's key -
109
+ its answers over the version's earlier ones. Returns the record as
110
+ stored. The pre-flight reads the workspace's own calibration unless one
111
+ is given, so a judge that counts more tokens per character than the rule
112
+ of thumb is refused on its own evidence."""
113
+ if calibration is None:
114
+ calibration = store.calibration(ws_dir)
115
+ rec = ask_states(states, key, post=post, calibration=calibration)
116
+ if rec is not None:
117
+ # a call over some of the kinds keeps the earlier readings for the others, as the answers are kept
118
+ lane = getattr(key, "lane", "system-one") # the LLM judge's records apart from System One's
119
+ earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane) or {}
120
+ rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})}
121
+ store.save(ws_dir, job_id, trial_name, rec, lane=lane)
122
+ return rec