nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/traces/otel.py ADDED
@@ -0,0 +1,608 @@
1
+ """OpenTelemetry span exports (OTLP/JSON), read as ``harbor-trace-v1`` traces.
2
+
3
+ A production trace export and Harbor's ``agent/trace.json`` describe the same
4
+ thing — spans with clocks and parentage — in two encodings. This reads the
5
+ first and writes the second, so a trace that never went through a benchmark
6
+ opens on the trial page and pools on ``/charts`` exactly as a recorded trial
7
+ does. Nothing here measures anything: every clock, id and count comes off the
8
+ wire document, and what the document does not say stays absent.
9
+
10
+ **Supported input format.** OTLP/JSON as the OpenTelemetry protocol 1.x JSON
11
+ encoding of ``ExportTraceServiceRequest`` — a top-level ``resourceSpans`` list,
12
+ each with ``resource.attributes``, ``scopeSpans[].scope`` and
13
+ ``scopeSpans[].spans[]`` carrying ``traceId``, ``spanId``, ``parentSpanId``,
14
+ ``name``, ``kind``, ``startTimeUnixNano``, ``endTimeUnixNano``, ``attributes``,
15
+ ``events`` and ``status``. Ids are the hex strings OTLP/JSON specifies and are
16
+ kept opaque; a base64 id would be kept opaque too. A file holds one request as
17
+ a JSON object, or one per line (what the collector's ``file`` exporter writes),
18
+ and nothing else. A ``schemaUrl`` is accepted when it is
19
+ an ``opentelemetry.io/schemas/1.*`` URL and refused otherwise. OTLP protobuf,
20
+ Jaeger and Zipkin JSON, and live collectors are out of scope.
21
+
22
+ **Semantic conventions.** OpenInference 0.1.x attributes are read where present,
23
+ and so are the Langfuse observation attributes (``langfuse.observation.*``) a
24
+ Langfuse-instrumented application's spans carry. Every other attribute is kept
25
+ verbatim, which is what keeps a generic span — no LLM attributes at all —
26
+ inspectable: its name, clock, parent, status, events, resource and attributes
27
+ all reach the detail pane.
28
+
29
+ Mapping, input → the fields the trial page reads
30
+ spanId / parentSpanId / traceId span_id / parent_span_id / trace_id
31
+ name name (and the label, unless tool.name)
32
+ startTimeUnixNano start_ns; nanoseconds, kept exact
33
+ endTimeUnixNano end_ns and duration_ns; a missing or
34
+ zero end is *incomplete*: no end, no
35
+ duration, timing_quality ``unknown``
36
+ and ``otel.incomplete: true``
37
+ parent not in this trace kept as a root with ``otel.orphan:
38
+ true``; parent_span_id stays as recorded
39
+ status.code ERROR ``otel.status: ERROR`` plus
40
+ ``otel.status_message``; never an
41
+ outcome — see below
42
+ events[] kept under ``events`` (name, time_ns,
43
+ attributes), summarised in
44
+ ``otel.events``; the first ``exception``
45
+ event's type and message are copied to
46
+ ``otel.exception.type`` / ``.message``
47
+ kind (OTLP SpanKind) ``otel.span_kind`` (INTERNAL, CLIENT, …)
48
+ resource.attributes merged into every span's attributes
49
+ (span attributes win on a clash)
50
+ scope.name / scope.version ``otel.scope.name`` / ``otel.scope.version``
51
+ openinference.span.kind LLM → llm_request, TOOL → tool, AGENT →
52
+ session at a root and subagent below;
53
+ CHAIN, RETRIEVER, EMBEDDING, RERANKER,
54
+ GUARDRAIL, EVALUATOR keep their
55
+ lower-cased name; no kind → ``span``
56
+ langfuse.observation.type generation → llm_request, tool → tool,
57
+ agent → as AGENT above; others as-is
58
+ llm.model_name kept; the importer reads the first one
59
+ as the trial's model
60
+ llm.token_count.prompt / completion tokens.input_tokens / output_tokens
61
+ llm.token_count.total tokens.total_tokens
62
+ llm.token_count.prompt_details.cache_read / cache_write
63
+ tokens.cache_read_tokens / cache_write_tokens
64
+ llm.token_count.completion_details.reasoning
65
+ tokens.reasoning_tokens
66
+ langfuse.observation.usage_details input / output / input_cached_tokens /
67
+ total → the same four token fields
68
+ input.value / output.value content.input / content.output on an LLM
69
+ or generic span, content.arguments /
70
+ content.result on a tool span; clipped
71
+ as clip() cuts it
72
+ tool.parameters content.arguments when a tool span has
73
+ no input.value
74
+ langfuse.observation.input / output as input.value / output.value
75
+ tool.name ``tool_name``, which is what the label
76
+ prefers over the span name
77
+
78
+ Usage is read **per span, on LLM spans only** (``llm_request`` after the kind
79
+ mapping), and ``totals.tokens`` is the sum over those spans. A container that
80
+ carries aggregated counts of its own — an AGENT or CHAIN span some
81
+ instrumentations stamp with the sum of their children — keeps them as
82
+ attributes, verbatim, and they are not read as usage: reading them would count
83
+ every call twice. A count the span does not carry is absent, never zero.
84
+
85
+ ``totals`` is computed once, here, by the trial page's own partition
86
+ (:func:`trace.breakdown`): every instant charged to one category at priority
87
+ idle > tool > reasoning > llm, so nested and concurrent spans never sum past
88
+ the clock. ``wall_ns`` is the trace's own window — first start to last end —
89
+ which is what the coverage strip then compares the spans against. Spans are
90
+ ``timing_quality: measured``: these are the producer's clocks, not this
91
+ reader's inference. Spans are ordered by start, out-of-order input included.
92
+
93
+ **What is not mapped, and what is lost.** OTLP ``links``, ``droppedAttributesCount``
94
+ and its siblings, ``traceState`` and ``flags`` are not read. The OpenTelemetry
95
+ GenAI conventions (``gen_ai.*``), OpenInference message lists
96
+ (``llm.input_messages.*``), embeddings, retrieval documents and
97
+ ``llm.invocation_parameters`` are kept as attributes and not interpreted. Usage
98
+ on a non-LLM span is not totalled (above). Content is clipped, and an export
99
+ that was itself clipped is clipped again. Nothing here says whether the trace *succeeded*: a benchmark outcome is
100
+ the verifier's to record, so a span in ERROR marks that span and the trial's
101
+ outcome stays unknown.
102
+
103
+ Redaction applies on the way in — attribute values, event attributes, status
104
+ messages, content and this reader's own error messages — because the importer
105
+ writes the result into a workspace. Stdlib only; no Flask.
106
+ """
107
+
108
+ from __future__ import annotations
109
+
110
+ import base64
111
+ import json
112
+ import os
113
+ import re
114
+
115
+ from nullius.redact import redact
116
+ from nullius.traces import trace
117
+
118
+ DEFAULT_MAX_CONTENT = 2000
119
+
120
+
121
+ def clip(text, limit: int) -> str:
122
+ """A span's text, cut at ``limit`` characters with the rest counted."""
123
+ if not isinstance(text, str):
124
+ text = json.dumps(text, ensure_ascii=False, default=str)
125
+ if limit and len(text) > limit:
126
+ return text[:limit] + f"\n… [{len(text) - limit} more characters]"
127
+ return text
128
+
129
+ SCHEMA = "harbor-trace-v1"
130
+ PRODUCER = "app/nullius/traces/otel.py"
131
+
132
+ # What this reader is written against. Named in the document it writes and in
133
+ # the fixtures' README, so a claim of support can be checked against a version.
134
+ FORMAT = "OTLP/JSON ExportTraceServiceRequest, opentelemetry-proto 1.x"
135
+ CONVENTIONS = "OpenInference semantic conventions 0.1.x"
136
+
137
+ # The schema URL OTLP lets a resource or scope carry. Only the 1.x line is
138
+ # claimed; anything else names a convention set this reader was not written for.
139
+ _SCHEMA_URL_RE = re.compile(r"^https://opentelemetry\.io/schemas/1\.\d+(\.\d+)?$")
140
+
141
+ # The span kinds the trial page renders (`trace.KIND_ORDER`) for the kinds the
142
+ # two conventions name. A kind absent here keeps its lower-cased name — the
143
+ # page paints an unknown kind neutral rather than dropping it.
144
+ _OPENINFERENCE_KINDS = {"LLM": "llm_request", "TOOL": "tool"}
145
+ _LANGFUSE_TYPES = {"generation": "llm_request", "tool": "tool"}
146
+ _AGENT_KINDS = ("AGENT", "agent")
147
+ GENERIC_KIND = "span"
148
+
149
+ # OTLP SpanKind, by number and by enum name; both spellings are legal JSON.
150
+ _SPAN_KINDS = ("UNSPECIFIED", "INTERNAL", "SERVER", "CLIENT", "PRODUCER", "CONSUMER")
151
+ _STATUS_CODES = ("UNSET", "OK", "ERROR")
152
+
153
+ # OpenInference token attributes → the token fields the views read.
154
+ _TOKEN_ATTRS = (
155
+ ("llm.token_count.prompt", "input_tokens"),
156
+ ("llm.token_count.completion", "output_tokens"),
157
+ ("llm.token_count.total", "total_tokens"),
158
+ ("llm.token_count.prompt_details.cache_read", "cache_read_tokens"),
159
+ ("llm.token_count.prompt_details.cache_write", "cache_write_tokens"),
160
+ ("llm.token_count.completion_details.reasoning", "reasoning_tokens"),
161
+ )
162
+ # Langfuse `usage_details` keys → the same fields.
163
+ _LANGFUSE_USAGE = (
164
+ ("input", "input_tokens"),
165
+ ("output", "output_tokens"),
166
+ ("input_cached_tokens", "cache_read_tokens"),
167
+ ("total", "total_tokens"),
168
+ )
169
+
170
+
171
+ class ReadError(ValueError):
172
+ """A file this reader refuses. The message is already redacted."""
173
+
174
+
175
+
176
+ def _int_or_none(value) -> int | None:
177
+ if isinstance(value, bool):
178
+ return None
179
+ if isinstance(value, int):
180
+ return value
181
+ if isinstance(value, str) and value.strip().lstrip("-").isdigit():
182
+ return int(value)
183
+ return None
184
+
185
+
186
+ def any_value(wire):
187
+ """One OTLP ``AnyValue`` as a plain JSON value; strings redacted."""
188
+ if not isinstance(wire, dict):
189
+ return None
190
+ if "stringValue" in wire:
191
+ return redact(str(wire["stringValue"]))
192
+ if "intValue" in wire:
193
+ parsed = _int_or_none(wire["intValue"])
194
+ return parsed if parsed is not None else redact(str(wire["intValue"]))
195
+ if "doubleValue" in wire:
196
+ try:
197
+ return float(wire["doubleValue"])
198
+ except (TypeError, ValueError):
199
+ return None
200
+ if "boolValue" in wire:
201
+ return bool(wire["boolValue"])
202
+ if "arrayValue" in wire:
203
+ values = (wire["arrayValue"] or {}).get("values") or []
204
+ return [any_value(v) for v in values if isinstance(v, dict)]
205
+ if "kvlistValue" in wire:
206
+ return attributes_of((wire["kvlistValue"] or {}).get("values") or [])
207
+ if "bytesValue" in wire:
208
+ try:
209
+ raw = base64.b64decode(str(wire["bytesValue"]), validate=True)
210
+ except (ValueError, TypeError):
211
+ return redact(str(wire["bytesValue"]))
212
+ return f"<{len(raw)} bytes>"
213
+ return None
214
+
215
+
216
+ def attributes_of(wire) -> dict:
217
+ """An OTLP ``KeyValue`` list as a dict. Keys are instrumentation names."""
218
+ out: dict = {}
219
+ for item in wire or []:
220
+ if not isinstance(item, dict) or not item.get("key"):
221
+ continue
222
+ out[str(item["key"])] = any_value(item.get("value"))
223
+ return out
224
+
225
+
226
+
227
+ def _requests_in(doc, where: str) -> list:
228
+ """The ``ExportTraceServiceRequest`` objects one decoded document holds."""
229
+ if isinstance(doc, dict) and isinstance(doc.get("resourceSpans"), list):
230
+ return [doc]
231
+ keys = sorted(doc)[:8] if isinstance(doc, dict) else type(doc).__name__
232
+ raise ReadError(redact(f"{where}: not an OTLP/JSON trace export "
233
+ f"(no resourceSpans; top level: {keys})"))
234
+
235
+
236
+ def _check_schema(request: dict, where: str) -> None:
237
+ for block in request.get("resourceSpans") or []:
238
+ if not isinstance(block, dict):
239
+ continue
240
+ urls = [block.get("schemaUrl")] + [
241
+ s.get("schemaUrl") for s in block.get("scopeSpans") or [] if isinstance(s, dict)]
242
+ for url in urls:
243
+ if url and not _SCHEMA_URL_RE.match(str(url)):
244
+ raise ReadError(redact(
245
+ f"{where}: unsupported schema {url!s}; this reader is written "
246
+ f"against {FORMAT}"))
247
+
248
+
249
+ def read_file(path: str) -> list:
250
+ """Every request in one file — a JSON object, or one object per line.
251
+
252
+ Refuses, with a redacted message, what it cannot read as OTLP/JSON: bad
253
+ JSON, a document without ``resourceSpans``, a schema URL outside the 1.x
254
+ line. Refusing is the whole contract — a file this half-reads would render
255
+ as a trace with spans missing and nothing saying so.
256
+ """
257
+ where = os.path.basename(path)
258
+ try:
259
+ with open(path, encoding="utf-8") as fh:
260
+ text = fh.read()
261
+ except OSError as exc:
262
+ raise ReadError(redact(f"{where}: {exc.strerror or exc}")) from None
263
+ try:
264
+ docs = [json.loads(text)]
265
+ except ValueError as whole:
266
+ lines = [line for line in text.splitlines() if line.strip()]
267
+ try:
268
+ docs = [json.loads(line) for line in lines] if lines else []
269
+ except ValueError:
270
+ raise ReadError(redact(f"{where}: not JSON ({whole})")) from None
271
+ if not docs:
272
+ raise ReadError(redact(f"{where}: not JSON ({whole})")) from None
273
+ requests = [req for doc in docs for req in _requests_in(doc, where)]
274
+ for req in requests:
275
+ _check_schema(req, where)
276
+ return requests
277
+
278
+
279
+ def looks_like_export(path: str) -> bool:
280
+ """Cheap first look: the key has to be in the file's head for it to be one."""
281
+ try:
282
+ with open(path, "rb") as fh:
283
+ head = fh.read(4096)
284
+ except OSError:
285
+ return False
286
+ return b"resourceSpans" in head
287
+
288
+
289
+
290
+ def flat_spans(request: dict) -> list:
291
+ """Every span in a request as a flat record carrying its resource and scope.
292
+
293
+ The record keeps the wire span under ``span`` and the decoded, redacted
294
+ resource and scope beside it; :func:`build` does the mapping.
295
+ """
296
+ out = []
297
+ for block in request.get("resourceSpans") or []:
298
+ if not isinstance(block, dict):
299
+ continue
300
+ resource = attributes_of((block.get("resource") or {}).get("attributes"))
301
+ for scoped in block.get("scopeSpans") or []:
302
+ if not isinstance(scoped, dict):
303
+ continue
304
+ scope = scoped.get("scope") or {}
305
+ for span in scoped.get("spans") or []:
306
+ if isinstance(span, dict):
307
+ out.append({"span": span, "resource": resource,
308
+ "scope": {"name": str(scope.get("name") or ""),
309
+ "version": str(scope.get("version") or "")}})
310
+ return out
311
+
312
+
313
+ def traces_of(records: list) -> dict:
314
+ """Flat records grouped by trace id, in first-seen order."""
315
+ out: dict = {}
316
+ for rec in records:
317
+ tid = str(rec["span"].get("traceId") or "")
318
+ out.setdefault(tid, []).append(rec)
319
+ return out
320
+
321
+
322
+
323
+ def _enum(value, names: tuple) -> str:
324
+ """An OTLP enum, given as its number or its ``PREFIX_NAME`` string."""
325
+ number = _int_or_none(value)
326
+ if number is not None:
327
+ return names[number] if 0 <= number < len(names) else str(number)
328
+ text = str(value or "").upper()
329
+ for prefix in ("SPAN_KIND_", "STATUS_CODE_"):
330
+ text = text[len(prefix):] if text.startswith(prefix) else text
331
+ return text or names[0]
332
+
333
+
334
+ def _kind_of(attributes: dict, is_root: bool) -> str:
335
+ """The trial page's kind for a span, from whichever convention names one."""
336
+ oi = attributes.get("openinference.span.kind")
337
+ lf = attributes.get("langfuse.observation.type")
338
+ for raw, table in ((oi, _OPENINFERENCE_KINDS), (lf, _LANGFUSE_TYPES)):
339
+ if not isinstance(raw, str) or not raw.strip():
340
+ continue
341
+ raw = raw.strip()
342
+ if raw in _AGENT_KINDS:
343
+ return "session" if is_root else "subagent"
344
+ return table.get(raw, raw.lower())
345
+ return GENERIC_KIND
346
+
347
+
348
+ def _tokens_of(attributes: dict) -> dict:
349
+ """Usage from the OpenInference counts, else from Langfuse's usage_details."""
350
+ tokens = {}
351
+ for key, field in _TOKEN_ATTRS:
352
+ value = _int_or_none(attributes.get(key))
353
+ if value is not None:
354
+ tokens[field] = value
355
+ if tokens:
356
+ return tokens
357
+ usage = attributes.get("langfuse.observation.usage_details")
358
+ if isinstance(usage, str):
359
+ try:
360
+ usage = json.loads(usage)
361
+ except ValueError:
362
+ usage = None
363
+ if isinstance(usage, dict):
364
+ for key, field in _LANGFUSE_USAGE:
365
+ value = _int_or_none(usage.get(key))
366
+ if value is not None:
367
+ tokens[field] = value
368
+ return tokens
369
+
370
+
371
+ def _text(value, limit: int) -> str:
372
+ if value is None or value == "":
373
+ return ""
374
+ return clip(value if isinstance(value, str) else json.dumps(value, ensure_ascii=False), limit)
375
+
376
+
377
+ def _content_of(attributes: dict, kind: str, limit: int) -> dict:
378
+ """input/output on a call or generic span, arguments/result on a tool."""
379
+ inp = attributes.get("input.value")
380
+ if inp is None:
381
+ inp = attributes.get("langfuse.observation.input")
382
+ if inp is None and kind == "tool":
383
+ inp = attributes.get("tool.parameters")
384
+ out = attributes.get("output.value")
385
+ if out is None:
386
+ out = attributes.get("langfuse.observation.output")
387
+ fields = ("arguments", "result") if kind == "tool" else ("input", "output")
388
+ content = {}
389
+ for field, value in zip(fields, (inp, out)):
390
+ text = _text(value, limit)
391
+ if text:
392
+ content[field] = text
393
+ return content
394
+
395
+
396
+ def _events_of(wire: list) -> list:
397
+ out = []
398
+ for ev in wire or []:
399
+ if not isinstance(ev, dict):
400
+ continue
401
+ out.append({
402
+ "name": redact(str(ev.get("name") or "")),
403
+ "time_ns": _int_or_none(ev.get("timeUnixNano")),
404
+ "attributes": attributes_of(ev.get("attributes")),
405
+ })
406
+ return out
407
+
408
+
409
+ def _span_of(rec: dict, is_root: bool, present: set, limit: int) -> dict:
410
+ """One wire span as a ``harbor-trace-v1`` span, children still empty."""
411
+ wire = rec["span"]
412
+ attributes = {**rec["resource"]}
413
+ if rec["scope"]["name"]:
414
+ attributes["otel.scope.name"] = rec["scope"]["name"]
415
+ if rec["scope"]["version"]:
416
+ attributes["otel.scope.version"] = rec["scope"]["version"]
417
+ attributes.update(attributes_of(wire.get("attributes")))
418
+ attributes["otel.span_kind"] = _enum(wire.get("kind"), _SPAN_KINDS)
419
+
420
+ status = wire.get("status") or {}
421
+ code = _enum(status.get("code"), _STATUS_CODES) if isinstance(status, dict) else "UNSET"
422
+ attributes["otel.status"] = code
423
+ if isinstance(status, dict) and status.get("message"):
424
+ attributes["otel.status_message"] = redact(str(status["message"]))
425
+
426
+ events = _events_of(wire.get("events"))
427
+ start_ns = _int_or_none(wire.get("startTimeUnixNano")) or 0
428
+ if events:
429
+ attributes["otel.events"] = "; ".join(
430
+ f"{ev['name']} +{(ev['time_ns'] - start_ns) // trace.NS_PER_MS}ms"
431
+ if ev["time_ns"] is not None else ev["name"] for ev in events)
432
+ exc = next((ev for ev in events if ev["name"] == "exception"), None)
433
+ if exc:
434
+ for key in ("exception.type", "exception.message"):
435
+ if exc["attributes"].get(key):
436
+ attributes["otel." + key] = exc["attributes"][key]
437
+
438
+ kind = _kind_of(attributes, is_root)
439
+ tool = attributes.get("tool.name")
440
+ if isinstance(tool, str) and tool.strip():
441
+ attributes["tool_name"] = tool.strip()
442
+
443
+ parent = str(wire.get("parentSpanId") or "")
444
+ if parent and parent not in present:
445
+ attributes["otel.orphan"] = True
446
+
447
+ span = {
448
+ "span_id": str(wire.get("spanId") or ""),
449
+ "trace_id": str(wire.get("traceId") or ""),
450
+ "name": redact(str(wire.get("name") or "")),
451
+ "kind": kind,
452
+ "start_ns": start_ns,
453
+ "timing_quality": "measured",
454
+ "source": "otel",
455
+ "attributes": attributes,
456
+ "children": [],
457
+ }
458
+ if parent:
459
+ span["parent_span_id"] = parent
460
+ end_ns = _int_or_none(wire.get("endTimeUnixNano"))
461
+ if end_ns:
462
+ span["end_ns"] = end_ns
463
+ span["duration_ns"] = end_ns - start_ns
464
+ else:
465
+ attributes["otel.incomplete"] = True
466
+ span["timing_quality"] = "unknown"
467
+ if kind == "llm_request":
468
+ tokens = _tokens_of(attributes)
469
+ if tokens:
470
+ span["tokens"] = tokens
471
+ content = _content_of(attributes, kind, limit)
472
+ if content:
473
+ span["content"] = content
474
+ if events:
475
+ span["events"] = events
476
+ return span
477
+
478
+
479
+
480
+ def _order(spans: list) -> list:
481
+ return sorted(spans, key=lambda s: (s["start_ns"], s["span_id"]))
482
+
483
+
484
+ def build(trace_id: str, records: list, limit: int = DEFAULT_MAX_CONTENT,
485
+ files: tuple = ()) -> dict:
486
+ """One trace's records as a ``harbor-trace-v1`` document.
487
+
488
+ Spans are ordered by start at every level; a span whose parent is not in
489
+ the trace becomes a root marked orphan; a span id seen twice keeps its
490
+ first occurrence and the duplicate is noted, since the page keys rows by id
491
+ and would otherwise silently drop one of them.
492
+ """
493
+ notes: list = []
494
+ present = {str(r["span"].get("spanId") or "") for r in records}
495
+ by_id: dict = {}
496
+ order: list = []
497
+ for rec in records:
498
+ sid = str(rec["span"].get("spanId") or "")
499
+ parent = str(rec["span"].get("parentSpanId") or "")
500
+ is_root = not parent or parent not in present
501
+ if sid in by_id:
502
+ notes.append(f"span id {sid} appears twice; the first occurrence is kept")
503
+ continue
504
+ by_id[sid] = _span_of(rec, is_root, present, limit)
505
+ order.append(sid)
506
+
507
+ roots = []
508
+ for sid in order:
509
+ span = by_id[sid]
510
+ parent = span.get("parent_span_id", "")
511
+ if parent and parent in by_id and parent != sid:
512
+ by_id[parent]["children"].append(span)
513
+ else:
514
+ roots.append(span)
515
+ # A parent cycle (a under b under a) leaves both spans off every root. JSON
516
+ # cannot express one in the tree this writes, so the loop is broken here:
517
+ # whatever no root reaches is surfaced as an orphan root, not lost.
518
+ reached: set = set()
519
+ stack = list(roots)
520
+ while stack:
521
+ span = stack.pop()
522
+ if span["span_id"] in reached:
523
+ continue
524
+ reached.add(span["span_id"])
525
+ stack.extend(span["children"])
526
+ for sid in order:
527
+ if sid not in reached:
528
+ span = by_id[sid]
529
+ by_id[span["parent_span_id"]]["children"].remove(span)
530
+ span["attributes"]["otel.orphan"] = True
531
+ notes.append(f"span {sid} sits in a parent cycle; kept as a root")
532
+ roots.append(span)
533
+ reached.update(s["span_id"] for s in trace.raw_spans_of([span]))
534
+ for span in by_id.values():
535
+ span["children"] = _order(span["children"])
536
+ roots = _order(roots)
537
+
538
+ starts = [s["start_ns"] for s in by_id.values()]
539
+ ends = [s["end_ns"] for s in by_id.values() if "end_ns" in s]
540
+ wall_ns = (max(ends) - min(starts)) if starts and ends and max(ends) > min(starts) else 0
541
+ orphans = sum(1 for s in by_id.values() if s["attributes"].get("otel.orphan"))
542
+ incomplete = sum(1 for s in by_id.values() if s["attributes"].get("otel.incomplete"))
543
+ if orphans:
544
+ notes.append(f"{orphans} span(s) name a parent this trace does not carry; kept as roots")
545
+ if incomplete:
546
+ notes.append(f"{incomplete} span(s) have no end; kept without a duration")
547
+
548
+ root_name = roots[0]["name"] if roots else ""
549
+ resource = records[0]["resource"] if records else {}
550
+ return {
551
+ "schema_version": SCHEMA,
552
+ "producer": PRODUCER,
553
+ "trial_name": root_name or trace_id,
554
+ "session_id": trace_id,
555
+ "agent": {
556
+ "name": str(resource.get("service.name") or ""),
557
+ "version": str(resource.get("service.version") or ""),
558
+ },
559
+ "clock": {
560
+ "source": "span start/end (Unix epoch nanoseconds, as exported)",
561
+ "note": "wall_ns is the trace's own window, first start to last end",
562
+ },
563
+ "spans": roots,
564
+ # The trial page's partition, run once here: idle > tool > reasoning >
565
+ # llm, every instant charged to one category. Tokens are the sum over
566
+ # LLM spans, the only ones usage was read on.
567
+ "totals": trace.breakdown(roots, wall_ns).as_totals(),
568
+ "import": {
569
+ "format": FORMAT,
570
+ "conventions": CONVENTIONS,
571
+ "trace_id": trace_id,
572
+ "files": list(files),
573
+ "notes": notes,
574
+ },
575
+ }
576
+
577
+
578
+ def model_of(document: dict) -> str:
579
+ """The first LLM span's model name, or "" — what the importer shows."""
580
+ for raw in trace.raw_spans_of(document.get("spans") or []):
581
+ if raw.get("kind") != "llm_request":
582
+ continue
583
+ attrs = raw.get("attributes") or {}
584
+ for key in ("llm.model_name", "langfuse.observation.model.name", "gen_ai.request.model"):
585
+ if isinstance(attrs.get(key), str) and attrs[key].strip():
586
+ return attrs[key].strip()
587
+ return ""
588
+
589
+
590
+ def read(paths: list, limit: int = DEFAULT_MAX_CONTENT) -> list:
591
+ """Every trace across *paths*, one document each, in first-seen order.
592
+
593
+ A trace may be split across files (a batch per export), which is why the
594
+ files are read together and grouped afterwards. A file that does not read
595
+ raises :class:`ReadError` — the caller decides whether that is fatal.
596
+ """
597
+ records = []
598
+ where: dict = {}
599
+ for path in paths:
600
+ for request in read_file(path):
601
+ for rec in flat_spans(request):
602
+ records.append(rec)
603
+ where.setdefault(str(rec["span"].get("traceId") or ""), []).append(
604
+ os.path.basename(path))
605
+ return [
606
+ build(tid, recs, limit, tuple(sorted(set(where.get(tid, ())))))
607
+ for tid, recs in traces_of(records).items()
608
+ ]