nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/traces/otel.py
ADDED
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
"""OpenTelemetry span exports (OTLP/JSON), read as ``harbor-trace-v1`` traces.
|
|
2
|
+
|
|
3
|
+
A production trace export and Harbor's ``agent/trace.json`` describe the same
|
|
4
|
+
thing — spans with clocks and parentage — in two encodings. This reads the
|
|
5
|
+
first and writes the second, so a trace that never went through a benchmark
|
|
6
|
+
opens on the trial page and pools on ``/charts`` exactly as a recorded trial
|
|
7
|
+
does. Nothing here measures anything: every clock, id and count comes off the
|
|
8
|
+
wire document, and what the document does not say stays absent.
|
|
9
|
+
|
|
10
|
+
**Supported input format.** OTLP/JSON as the OpenTelemetry protocol 1.x JSON
|
|
11
|
+
encoding of ``ExportTraceServiceRequest`` — a top-level ``resourceSpans`` list,
|
|
12
|
+
each with ``resource.attributes``, ``scopeSpans[].scope`` and
|
|
13
|
+
``scopeSpans[].spans[]`` carrying ``traceId``, ``spanId``, ``parentSpanId``,
|
|
14
|
+
``name``, ``kind``, ``startTimeUnixNano``, ``endTimeUnixNano``, ``attributes``,
|
|
15
|
+
``events`` and ``status``. Ids are the hex strings OTLP/JSON specifies and are
|
|
16
|
+
kept opaque; a base64 id would be kept opaque too. A file holds one request as
|
|
17
|
+
a JSON object, or one per line (what the collector's ``file`` exporter writes),
|
|
18
|
+
and nothing else. A ``schemaUrl`` is accepted when it is
|
|
19
|
+
an ``opentelemetry.io/schemas/1.*`` URL and refused otherwise. OTLP protobuf,
|
|
20
|
+
Jaeger and Zipkin JSON, and live collectors are out of scope.
|
|
21
|
+
|
|
22
|
+
**Semantic conventions.** OpenInference 0.1.x attributes are read where present,
|
|
23
|
+
and so are the Langfuse observation attributes (``langfuse.observation.*``) a
|
|
24
|
+
Langfuse-instrumented application's spans carry. Every other attribute is kept
|
|
25
|
+
verbatim, which is what keeps a generic span — no LLM attributes at all —
|
|
26
|
+
inspectable: its name, clock, parent, status, events, resource and attributes
|
|
27
|
+
all reach the detail pane.
|
|
28
|
+
|
|
29
|
+
Mapping, input → the fields the trial page reads
|
|
30
|
+
spanId / parentSpanId / traceId span_id / parent_span_id / trace_id
|
|
31
|
+
name name (and the label, unless tool.name)
|
|
32
|
+
startTimeUnixNano start_ns; nanoseconds, kept exact
|
|
33
|
+
endTimeUnixNano end_ns and duration_ns; a missing or
|
|
34
|
+
zero end is *incomplete*: no end, no
|
|
35
|
+
duration, timing_quality ``unknown``
|
|
36
|
+
and ``otel.incomplete: true``
|
|
37
|
+
parent not in this trace kept as a root with ``otel.orphan:
|
|
38
|
+
true``; parent_span_id stays as recorded
|
|
39
|
+
status.code ERROR ``otel.status: ERROR`` plus
|
|
40
|
+
``otel.status_message``; never an
|
|
41
|
+
outcome — see below
|
|
42
|
+
events[] kept under ``events`` (name, time_ns,
|
|
43
|
+
attributes), summarised in
|
|
44
|
+
``otel.events``; the first ``exception``
|
|
45
|
+
event's type and message are copied to
|
|
46
|
+
``otel.exception.type`` / ``.message``
|
|
47
|
+
kind (OTLP SpanKind) ``otel.span_kind`` (INTERNAL, CLIENT, …)
|
|
48
|
+
resource.attributes merged into every span's attributes
|
|
49
|
+
(span attributes win on a clash)
|
|
50
|
+
scope.name / scope.version ``otel.scope.name`` / ``otel.scope.version``
|
|
51
|
+
openinference.span.kind LLM → llm_request, TOOL → tool, AGENT →
|
|
52
|
+
session at a root and subagent below;
|
|
53
|
+
CHAIN, RETRIEVER, EMBEDDING, RERANKER,
|
|
54
|
+
GUARDRAIL, EVALUATOR keep their
|
|
55
|
+
lower-cased name; no kind → ``span``
|
|
56
|
+
langfuse.observation.type generation → llm_request, tool → tool,
|
|
57
|
+
agent → as AGENT above; others as-is
|
|
58
|
+
llm.model_name kept; the importer reads the first one
|
|
59
|
+
as the trial's model
|
|
60
|
+
llm.token_count.prompt / completion tokens.input_tokens / output_tokens
|
|
61
|
+
llm.token_count.total tokens.total_tokens
|
|
62
|
+
llm.token_count.prompt_details.cache_read / cache_write
|
|
63
|
+
tokens.cache_read_tokens / cache_write_tokens
|
|
64
|
+
llm.token_count.completion_details.reasoning
|
|
65
|
+
tokens.reasoning_tokens
|
|
66
|
+
langfuse.observation.usage_details input / output / input_cached_tokens /
|
|
67
|
+
total → the same four token fields
|
|
68
|
+
input.value / output.value content.input / content.output on an LLM
|
|
69
|
+
or generic span, content.arguments /
|
|
70
|
+
content.result on a tool span; clipped
|
|
71
|
+
as clip() cuts it
|
|
72
|
+
tool.parameters content.arguments when a tool span has
|
|
73
|
+
no input.value
|
|
74
|
+
langfuse.observation.input / output as input.value / output.value
|
|
75
|
+
tool.name ``tool_name``, which is what the label
|
|
76
|
+
prefers over the span name
|
|
77
|
+
|
|
78
|
+
Usage is read **per span, on LLM spans only** (``llm_request`` after the kind
|
|
79
|
+
mapping), and ``totals.tokens`` is the sum over those spans. A container that
|
|
80
|
+
carries aggregated counts of its own — an AGENT or CHAIN span some
|
|
81
|
+
instrumentations stamp with the sum of their children — keeps them as
|
|
82
|
+
attributes, verbatim, and they are not read as usage: reading them would count
|
|
83
|
+
every call twice. A count the span does not carry is absent, never zero.
|
|
84
|
+
|
|
85
|
+
``totals`` is computed once, here, by the trial page's own partition
|
|
86
|
+
(:func:`trace.breakdown`): every instant charged to one category at priority
|
|
87
|
+
idle > tool > reasoning > llm, so nested and concurrent spans never sum past
|
|
88
|
+
the clock. ``wall_ns`` is the trace's own window — first start to last end —
|
|
89
|
+
which is what the coverage strip then compares the spans against. Spans are
|
|
90
|
+
``timing_quality: measured``: these are the producer's clocks, not this
|
|
91
|
+
reader's inference. Spans are ordered by start, out-of-order input included.
|
|
92
|
+
|
|
93
|
+
**What is not mapped, and what is lost.** OTLP ``links``, ``droppedAttributesCount``
|
|
94
|
+
and its siblings, ``traceState`` and ``flags`` are not read. The OpenTelemetry
|
|
95
|
+
GenAI conventions (``gen_ai.*``), OpenInference message lists
|
|
96
|
+
(``llm.input_messages.*``), embeddings, retrieval documents and
|
|
97
|
+
``llm.invocation_parameters`` are kept as attributes and not interpreted. Usage
|
|
98
|
+
on a non-LLM span is not totalled (above). Content is clipped, and an export
|
|
99
|
+
that was itself clipped is clipped again. Nothing here says whether the trace *succeeded*: a benchmark outcome is
|
|
100
|
+
the verifier's to record, so a span in ERROR marks that span and the trial's
|
|
101
|
+
outcome stays unknown.
|
|
102
|
+
|
|
103
|
+
Redaction applies on the way in — attribute values, event attributes, status
|
|
104
|
+
messages, content and this reader's own error messages — because the importer
|
|
105
|
+
writes the result into a workspace. Stdlib only; no Flask.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
from __future__ import annotations
|
|
109
|
+
|
|
110
|
+
import base64
|
|
111
|
+
import json
|
|
112
|
+
import os
|
|
113
|
+
import re
|
|
114
|
+
|
|
115
|
+
from nullius.redact import redact
|
|
116
|
+
from nullius.traces import trace
|
|
117
|
+
|
|
118
|
+
DEFAULT_MAX_CONTENT = 2000
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def clip(text, limit: int) -> str:
|
|
122
|
+
"""A span's text, cut at ``limit`` characters with the rest counted."""
|
|
123
|
+
if not isinstance(text, str):
|
|
124
|
+
text = json.dumps(text, ensure_ascii=False, default=str)
|
|
125
|
+
if limit and len(text) > limit:
|
|
126
|
+
return text[:limit] + f"\n… [{len(text) - limit} more characters]"
|
|
127
|
+
return text
|
|
128
|
+
|
|
129
|
+
SCHEMA = "harbor-trace-v1"
|
|
130
|
+
PRODUCER = "app/nullius/traces/otel.py"
|
|
131
|
+
|
|
132
|
+
# What this reader is written against. Named in the document it writes and in
|
|
133
|
+
# the fixtures' README, so a claim of support can be checked against a version.
|
|
134
|
+
FORMAT = "OTLP/JSON ExportTraceServiceRequest, opentelemetry-proto 1.x"
|
|
135
|
+
CONVENTIONS = "OpenInference semantic conventions 0.1.x"
|
|
136
|
+
|
|
137
|
+
# The schema URL OTLP lets a resource or scope carry. Only the 1.x line is
|
|
138
|
+
# claimed; anything else names a convention set this reader was not written for.
|
|
139
|
+
_SCHEMA_URL_RE = re.compile(r"^https://opentelemetry\.io/schemas/1\.\d+(\.\d+)?$")
|
|
140
|
+
|
|
141
|
+
# The span kinds the trial page renders (`trace.KIND_ORDER`) for the kinds the
|
|
142
|
+
# two conventions name. A kind absent here keeps its lower-cased name — the
|
|
143
|
+
# page paints an unknown kind neutral rather than dropping it.
|
|
144
|
+
_OPENINFERENCE_KINDS = {"LLM": "llm_request", "TOOL": "tool"}
|
|
145
|
+
_LANGFUSE_TYPES = {"generation": "llm_request", "tool": "tool"}
|
|
146
|
+
_AGENT_KINDS = ("AGENT", "agent")
|
|
147
|
+
GENERIC_KIND = "span"
|
|
148
|
+
|
|
149
|
+
# OTLP SpanKind, by number and by enum name; both spellings are legal JSON.
|
|
150
|
+
_SPAN_KINDS = ("UNSPECIFIED", "INTERNAL", "SERVER", "CLIENT", "PRODUCER", "CONSUMER")
|
|
151
|
+
_STATUS_CODES = ("UNSET", "OK", "ERROR")
|
|
152
|
+
|
|
153
|
+
# OpenInference token attributes → the token fields the views read.
|
|
154
|
+
_TOKEN_ATTRS = (
|
|
155
|
+
("llm.token_count.prompt", "input_tokens"),
|
|
156
|
+
("llm.token_count.completion", "output_tokens"),
|
|
157
|
+
("llm.token_count.total", "total_tokens"),
|
|
158
|
+
("llm.token_count.prompt_details.cache_read", "cache_read_tokens"),
|
|
159
|
+
("llm.token_count.prompt_details.cache_write", "cache_write_tokens"),
|
|
160
|
+
("llm.token_count.completion_details.reasoning", "reasoning_tokens"),
|
|
161
|
+
)
|
|
162
|
+
# Langfuse `usage_details` keys → the same fields.
|
|
163
|
+
_LANGFUSE_USAGE = (
|
|
164
|
+
("input", "input_tokens"),
|
|
165
|
+
("output", "output_tokens"),
|
|
166
|
+
("input_cached_tokens", "cache_read_tokens"),
|
|
167
|
+
("total", "total_tokens"),
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class ReadError(ValueError):
|
|
172
|
+
"""A file this reader refuses. The message is already redacted."""
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _int_or_none(value) -> int | None:
|
|
177
|
+
if isinstance(value, bool):
|
|
178
|
+
return None
|
|
179
|
+
if isinstance(value, int):
|
|
180
|
+
return value
|
|
181
|
+
if isinstance(value, str) and value.strip().lstrip("-").isdigit():
|
|
182
|
+
return int(value)
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def any_value(wire):
|
|
187
|
+
"""One OTLP ``AnyValue`` as a plain JSON value; strings redacted."""
|
|
188
|
+
if not isinstance(wire, dict):
|
|
189
|
+
return None
|
|
190
|
+
if "stringValue" in wire:
|
|
191
|
+
return redact(str(wire["stringValue"]))
|
|
192
|
+
if "intValue" in wire:
|
|
193
|
+
parsed = _int_or_none(wire["intValue"])
|
|
194
|
+
return parsed if parsed is not None else redact(str(wire["intValue"]))
|
|
195
|
+
if "doubleValue" in wire:
|
|
196
|
+
try:
|
|
197
|
+
return float(wire["doubleValue"])
|
|
198
|
+
except (TypeError, ValueError):
|
|
199
|
+
return None
|
|
200
|
+
if "boolValue" in wire:
|
|
201
|
+
return bool(wire["boolValue"])
|
|
202
|
+
if "arrayValue" in wire:
|
|
203
|
+
values = (wire["arrayValue"] or {}).get("values") or []
|
|
204
|
+
return [any_value(v) for v in values if isinstance(v, dict)]
|
|
205
|
+
if "kvlistValue" in wire:
|
|
206
|
+
return attributes_of((wire["kvlistValue"] or {}).get("values") or [])
|
|
207
|
+
if "bytesValue" in wire:
|
|
208
|
+
try:
|
|
209
|
+
raw = base64.b64decode(str(wire["bytesValue"]), validate=True)
|
|
210
|
+
except (ValueError, TypeError):
|
|
211
|
+
return redact(str(wire["bytesValue"]))
|
|
212
|
+
return f"<{len(raw)} bytes>"
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def attributes_of(wire) -> dict:
|
|
217
|
+
"""An OTLP ``KeyValue`` list as a dict. Keys are instrumentation names."""
|
|
218
|
+
out: dict = {}
|
|
219
|
+
for item in wire or []:
|
|
220
|
+
if not isinstance(item, dict) or not item.get("key"):
|
|
221
|
+
continue
|
|
222
|
+
out[str(item["key"])] = any_value(item.get("value"))
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _requests_in(doc, where: str) -> list:
|
|
228
|
+
"""The ``ExportTraceServiceRequest`` objects one decoded document holds."""
|
|
229
|
+
if isinstance(doc, dict) and isinstance(doc.get("resourceSpans"), list):
|
|
230
|
+
return [doc]
|
|
231
|
+
keys = sorted(doc)[:8] if isinstance(doc, dict) else type(doc).__name__
|
|
232
|
+
raise ReadError(redact(f"{where}: not an OTLP/JSON trace export "
|
|
233
|
+
f"(no resourceSpans; top level: {keys})"))
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _check_schema(request: dict, where: str) -> None:
|
|
237
|
+
for block in request.get("resourceSpans") or []:
|
|
238
|
+
if not isinstance(block, dict):
|
|
239
|
+
continue
|
|
240
|
+
urls = [block.get("schemaUrl")] + [
|
|
241
|
+
s.get("schemaUrl") for s in block.get("scopeSpans") or [] if isinstance(s, dict)]
|
|
242
|
+
for url in urls:
|
|
243
|
+
if url and not _SCHEMA_URL_RE.match(str(url)):
|
|
244
|
+
raise ReadError(redact(
|
|
245
|
+
f"{where}: unsupported schema {url!s}; this reader is written "
|
|
246
|
+
f"against {FORMAT}"))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def read_file(path: str) -> list:
|
|
250
|
+
"""Every request in one file — a JSON object, or one object per line.
|
|
251
|
+
|
|
252
|
+
Refuses, with a redacted message, what it cannot read as OTLP/JSON: bad
|
|
253
|
+
JSON, a document without ``resourceSpans``, a schema URL outside the 1.x
|
|
254
|
+
line. Refusing is the whole contract — a file this half-reads would render
|
|
255
|
+
as a trace with spans missing and nothing saying so.
|
|
256
|
+
"""
|
|
257
|
+
where = os.path.basename(path)
|
|
258
|
+
try:
|
|
259
|
+
with open(path, encoding="utf-8") as fh:
|
|
260
|
+
text = fh.read()
|
|
261
|
+
except OSError as exc:
|
|
262
|
+
raise ReadError(redact(f"{where}: {exc.strerror or exc}")) from None
|
|
263
|
+
try:
|
|
264
|
+
docs = [json.loads(text)]
|
|
265
|
+
except ValueError as whole:
|
|
266
|
+
lines = [line for line in text.splitlines() if line.strip()]
|
|
267
|
+
try:
|
|
268
|
+
docs = [json.loads(line) for line in lines] if lines else []
|
|
269
|
+
except ValueError:
|
|
270
|
+
raise ReadError(redact(f"{where}: not JSON ({whole})")) from None
|
|
271
|
+
if not docs:
|
|
272
|
+
raise ReadError(redact(f"{where}: not JSON ({whole})")) from None
|
|
273
|
+
requests = [req for doc in docs for req in _requests_in(doc, where)]
|
|
274
|
+
for req in requests:
|
|
275
|
+
_check_schema(req, where)
|
|
276
|
+
return requests
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def looks_like_export(path: str) -> bool:
|
|
280
|
+
"""Cheap first look: the key has to be in the file's head for it to be one."""
|
|
281
|
+
try:
|
|
282
|
+
with open(path, "rb") as fh:
|
|
283
|
+
head = fh.read(4096)
|
|
284
|
+
except OSError:
|
|
285
|
+
return False
|
|
286
|
+
return b"resourceSpans" in head
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def flat_spans(request: dict) -> list:
|
|
291
|
+
"""Every span in a request as a flat record carrying its resource and scope.
|
|
292
|
+
|
|
293
|
+
The record keeps the wire span under ``span`` and the decoded, redacted
|
|
294
|
+
resource and scope beside it; :func:`build` does the mapping.
|
|
295
|
+
"""
|
|
296
|
+
out = []
|
|
297
|
+
for block in request.get("resourceSpans") or []:
|
|
298
|
+
if not isinstance(block, dict):
|
|
299
|
+
continue
|
|
300
|
+
resource = attributes_of((block.get("resource") or {}).get("attributes"))
|
|
301
|
+
for scoped in block.get("scopeSpans") or []:
|
|
302
|
+
if not isinstance(scoped, dict):
|
|
303
|
+
continue
|
|
304
|
+
scope = scoped.get("scope") or {}
|
|
305
|
+
for span in scoped.get("spans") or []:
|
|
306
|
+
if isinstance(span, dict):
|
|
307
|
+
out.append({"span": span, "resource": resource,
|
|
308
|
+
"scope": {"name": str(scope.get("name") or ""),
|
|
309
|
+
"version": str(scope.get("version") or "")}})
|
|
310
|
+
return out
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def traces_of(records: list) -> dict:
|
|
314
|
+
"""Flat records grouped by trace id, in first-seen order."""
|
|
315
|
+
out: dict = {}
|
|
316
|
+
for rec in records:
|
|
317
|
+
tid = str(rec["span"].get("traceId") or "")
|
|
318
|
+
out.setdefault(tid, []).append(rec)
|
|
319
|
+
return out
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _enum(value, names: tuple) -> str:
|
|
324
|
+
"""An OTLP enum, given as its number or its ``PREFIX_NAME`` string."""
|
|
325
|
+
number = _int_or_none(value)
|
|
326
|
+
if number is not None:
|
|
327
|
+
return names[number] if 0 <= number < len(names) else str(number)
|
|
328
|
+
text = str(value or "").upper()
|
|
329
|
+
for prefix in ("SPAN_KIND_", "STATUS_CODE_"):
|
|
330
|
+
text = text[len(prefix):] if text.startswith(prefix) else text
|
|
331
|
+
return text or names[0]
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _kind_of(attributes: dict, is_root: bool) -> str:
|
|
335
|
+
"""The trial page's kind for a span, from whichever convention names one."""
|
|
336
|
+
oi = attributes.get("openinference.span.kind")
|
|
337
|
+
lf = attributes.get("langfuse.observation.type")
|
|
338
|
+
for raw, table in ((oi, _OPENINFERENCE_KINDS), (lf, _LANGFUSE_TYPES)):
|
|
339
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
340
|
+
continue
|
|
341
|
+
raw = raw.strip()
|
|
342
|
+
if raw in _AGENT_KINDS:
|
|
343
|
+
return "session" if is_root else "subagent"
|
|
344
|
+
return table.get(raw, raw.lower())
|
|
345
|
+
return GENERIC_KIND
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _tokens_of(attributes: dict) -> dict:
|
|
349
|
+
"""Usage from the OpenInference counts, else from Langfuse's usage_details."""
|
|
350
|
+
tokens = {}
|
|
351
|
+
for key, field in _TOKEN_ATTRS:
|
|
352
|
+
value = _int_or_none(attributes.get(key))
|
|
353
|
+
if value is not None:
|
|
354
|
+
tokens[field] = value
|
|
355
|
+
if tokens:
|
|
356
|
+
return tokens
|
|
357
|
+
usage = attributes.get("langfuse.observation.usage_details")
|
|
358
|
+
if isinstance(usage, str):
|
|
359
|
+
try:
|
|
360
|
+
usage = json.loads(usage)
|
|
361
|
+
except ValueError:
|
|
362
|
+
usage = None
|
|
363
|
+
if isinstance(usage, dict):
|
|
364
|
+
for key, field in _LANGFUSE_USAGE:
|
|
365
|
+
value = _int_or_none(usage.get(key))
|
|
366
|
+
if value is not None:
|
|
367
|
+
tokens[field] = value
|
|
368
|
+
return tokens
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _text(value, limit: int) -> str:
|
|
372
|
+
if value is None or value == "":
|
|
373
|
+
return ""
|
|
374
|
+
return clip(value if isinstance(value, str) else json.dumps(value, ensure_ascii=False), limit)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _content_of(attributes: dict, kind: str, limit: int) -> dict:
|
|
378
|
+
"""input/output on a call or generic span, arguments/result on a tool."""
|
|
379
|
+
inp = attributes.get("input.value")
|
|
380
|
+
if inp is None:
|
|
381
|
+
inp = attributes.get("langfuse.observation.input")
|
|
382
|
+
if inp is None and kind == "tool":
|
|
383
|
+
inp = attributes.get("tool.parameters")
|
|
384
|
+
out = attributes.get("output.value")
|
|
385
|
+
if out is None:
|
|
386
|
+
out = attributes.get("langfuse.observation.output")
|
|
387
|
+
fields = ("arguments", "result") if kind == "tool" else ("input", "output")
|
|
388
|
+
content = {}
|
|
389
|
+
for field, value in zip(fields, (inp, out)):
|
|
390
|
+
text = _text(value, limit)
|
|
391
|
+
if text:
|
|
392
|
+
content[field] = text
|
|
393
|
+
return content
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _events_of(wire: list) -> list:
|
|
397
|
+
out = []
|
|
398
|
+
for ev in wire or []:
|
|
399
|
+
if not isinstance(ev, dict):
|
|
400
|
+
continue
|
|
401
|
+
out.append({
|
|
402
|
+
"name": redact(str(ev.get("name") or "")),
|
|
403
|
+
"time_ns": _int_or_none(ev.get("timeUnixNano")),
|
|
404
|
+
"attributes": attributes_of(ev.get("attributes")),
|
|
405
|
+
})
|
|
406
|
+
return out
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _span_of(rec: dict, is_root: bool, present: set, limit: int) -> dict:
|
|
410
|
+
"""One wire span as a ``harbor-trace-v1`` span, children still empty."""
|
|
411
|
+
wire = rec["span"]
|
|
412
|
+
attributes = {**rec["resource"]}
|
|
413
|
+
if rec["scope"]["name"]:
|
|
414
|
+
attributes["otel.scope.name"] = rec["scope"]["name"]
|
|
415
|
+
if rec["scope"]["version"]:
|
|
416
|
+
attributes["otel.scope.version"] = rec["scope"]["version"]
|
|
417
|
+
attributes.update(attributes_of(wire.get("attributes")))
|
|
418
|
+
attributes["otel.span_kind"] = _enum(wire.get("kind"), _SPAN_KINDS)
|
|
419
|
+
|
|
420
|
+
status = wire.get("status") or {}
|
|
421
|
+
code = _enum(status.get("code"), _STATUS_CODES) if isinstance(status, dict) else "UNSET"
|
|
422
|
+
attributes["otel.status"] = code
|
|
423
|
+
if isinstance(status, dict) and status.get("message"):
|
|
424
|
+
attributes["otel.status_message"] = redact(str(status["message"]))
|
|
425
|
+
|
|
426
|
+
events = _events_of(wire.get("events"))
|
|
427
|
+
start_ns = _int_or_none(wire.get("startTimeUnixNano")) or 0
|
|
428
|
+
if events:
|
|
429
|
+
attributes["otel.events"] = "; ".join(
|
|
430
|
+
f"{ev['name']} +{(ev['time_ns'] - start_ns) // trace.NS_PER_MS}ms"
|
|
431
|
+
if ev["time_ns"] is not None else ev["name"] for ev in events)
|
|
432
|
+
exc = next((ev for ev in events if ev["name"] == "exception"), None)
|
|
433
|
+
if exc:
|
|
434
|
+
for key in ("exception.type", "exception.message"):
|
|
435
|
+
if exc["attributes"].get(key):
|
|
436
|
+
attributes["otel." + key] = exc["attributes"][key]
|
|
437
|
+
|
|
438
|
+
kind = _kind_of(attributes, is_root)
|
|
439
|
+
tool = attributes.get("tool.name")
|
|
440
|
+
if isinstance(tool, str) and tool.strip():
|
|
441
|
+
attributes["tool_name"] = tool.strip()
|
|
442
|
+
|
|
443
|
+
parent = str(wire.get("parentSpanId") or "")
|
|
444
|
+
if parent and parent not in present:
|
|
445
|
+
attributes["otel.orphan"] = True
|
|
446
|
+
|
|
447
|
+
span = {
|
|
448
|
+
"span_id": str(wire.get("spanId") or ""),
|
|
449
|
+
"trace_id": str(wire.get("traceId") or ""),
|
|
450
|
+
"name": redact(str(wire.get("name") or "")),
|
|
451
|
+
"kind": kind,
|
|
452
|
+
"start_ns": start_ns,
|
|
453
|
+
"timing_quality": "measured",
|
|
454
|
+
"source": "otel",
|
|
455
|
+
"attributes": attributes,
|
|
456
|
+
"children": [],
|
|
457
|
+
}
|
|
458
|
+
if parent:
|
|
459
|
+
span["parent_span_id"] = parent
|
|
460
|
+
end_ns = _int_or_none(wire.get("endTimeUnixNano"))
|
|
461
|
+
if end_ns:
|
|
462
|
+
span["end_ns"] = end_ns
|
|
463
|
+
span["duration_ns"] = end_ns - start_ns
|
|
464
|
+
else:
|
|
465
|
+
attributes["otel.incomplete"] = True
|
|
466
|
+
span["timing_quality"] = "unknown"
|
|
467
|
+
if kind == "llm_request":
|
|
468
|
+
tokens = _tokens_of(attributes)
|
|
469
|
+
if tokens:
|
|
470
|
+
span["tokens"] = tokens
|
|
471
|
+
content = _content_of(attributes, kind, limit)
|
|
472
|
+
if content:
|
|
473
|
+
span["content"] = content
|
|
474
|
+
if events:
|
|
475
|
+
span["events"] = events
|
|
476
|
+
return span
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _order(spans: list) -> list:
|
|
481
|
+
return sorted(spans, key=lambda s: (s["start_ns"], s["span_id"]))
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def build(trace_id: str, records: list, limit: int = DEFAULT_MAX_CONTENT,
|
|
485
|
+
files: tuple = ()) -> dict:
|
|
486
|
+
"""One trace's records as a ``harbor-trace-v1`` document.
|
|
487
|
+
|
|
488
|
+
Spans are ordered by start at every level; a span whose parent is not in
|
|
489
|
+
the trace becomes a root marked orphan; a span id seen twice keeps its
|
|
490
|
+
first occurrence and the duplicate is noted, since the page keys rows by id
|
|
491
|
+
and would otherwise silently drop one of them.
|
|
492
|
+
"""
|
|
493
|
+
notes: list = []
|
|
494
|
+
present = {str(r["span"].get("spanId") or "") for r in records}
|
|
495
|
+
by_id: dict = {}
|
|
496
|
+
order: list = []
|
|
497
|
+
for rec in records:
|
|
498
|
+
sid = str(rec["span"].get("spanId") or "")
|
|
499
|
+
parent = str(rec["span"].get("parentSpanId") or "")
|
|
500
|
+
is_root = not parent or parent not in present
|
|
501
|
+
if sid in by_id:
|
|
502
|
+
notes.append(f"span id {sid} appears twice; the first occurrence is kept")
|
|
503
|
+
continue
|
|
504
|
+
by_id[sid] = _span_of(rec, is_root, present, limit)
|
|
505
|
+
order.append(sid)
|
|
506
|
+
|
|
507
|
+
roots = []
|
|
508
|
+
for sid in order:
|
|
509
|
+
span = by_id[sid]
|
|
510
|
+
parent = span.get("parent_span_id", "")
|
|
511
|
+
if parent and parent in by_id and parent != sid:
|
|
512
|
+
by_id[parent]["children"].append(span)
|
|
513
|
+
else:
|
|
514
|
+
roots.append(span)
|
|
515
|
+
# A parent cycle (a under b under a) leaves both spans off every root. JSON
|
|
516
|
+
# cannot express one in the tree this writes, so the loop is broken here:
|
|
517
|
+
# whatever no root reaches is surfaced as an orphan root, not lost.
|
|
518
|
+
reached: set = set()
|
|
519
|
+
stack = list(roots)
|
|
520
|
+
while stack:
|
|
521
|
+
span = stack.pop()
|
|
522
|
+
if span["span_id"] in reached:
|
|
523
|
+
continue
|
|
524
|
+
reached.add(span["span_id"])
|
|
525
|
+
stack.extend(span["children"])
|
|
526
|
+
for sid in order:
|
|
527
|
+
if sid not in reached:
|
|
528
|
+
span = by_id[sid]
|
|
529
|
+
by_id[span["parent_span_id"]]["children"].remove(span)
|
|
530
|
+
span["attributes"]["otel.orphan"] = True
|
|
531
|
+
notes.append(f"span {sid} sits in a parent cycle; kept as a root")
|
|
532
|
+
roots.append(span)
|
|
533
|
+
reached.update(s["span_id"] for s in trace.raw_spans_of([span]))
|
|
534
|
+
for span in by_id.values():
|
|
535
|
+
span["children"] = _order(span["children"])
|
|
536
|
+
roots = _order(roots)
|
|
537
|
+
|
|
538
|
+
starts = [s["start_ns"] for s in by_id.values()]
|
|
539
|
+
ends = [s["end_ns"] for s in by_id.values() if "end_ns" in s]
|
|
540
|
+
wall_ns = (max(ends) - min(starts)) if starts and ends and max(ends) > min(starts) else 0
|
|
541
|
+
orphans = sum(1 for s in by_id.values() if s["attributes"].get("otel.orphan"))
|
|
542
|
+
incomplete = sum(1 for s in by_id.values() if s["attributes"].get("otel.incomplete"))
|
|
543
|
+
if orphans:
|
|
544
|
+
notes.append(f"{orphans} span(s) name a parent this trace does not carry; kept as roots")
|
|
545
|
+
if incomplete:
|
|
546
|
+
notes.append(f"{incomplete} span(s) have no end; kept without a duration")
|
|
547
|
+
|
|
548
|
+
root_name = roots[0]["name"] if roots else ""
|
|
549
|
+
resource = records[0]["resource"] if records else {}
|
|
550
|
+
return {
|
|
551
|
+
"schema_version": SCHEMA,
|
|
552
|
+
"producer": PRODUCER,
|
|
553
|
+
"trial_name": root_name or trace_id,
|
|
554
|
+
"session_id": trace_id,
|
|
555
|
+
"agent": {
|
|
556
|
+
"name": str(resource.get("service.name") or ""),
|
|
557
|
+
"version": str(resource.get("service.version") or ""),
|
|
558
|
+
},
|
|
559
|
+
"clock": {
|
|
560
|
+
"source": "span start/end (Unix epoch nanoseconds, as exported)",
|
|
561
|
+
"note": "wall_ns is the trace's own window, first start to last end",
|
|
562
|
+
},
|
|
563
|
+
"spans": roots,
|
|
564
|
+
# The trial page's partition, run once here: idle > tool > reasoning >
|
|
565
|
+
# llm, every instant charged to one category. Tokens are the sum over
|
|
566
|
+
# LLM spans, the only ones usage was read on.
|
|
567
|
+
"totals": trace.breakdown(roots, wall_ns).as_totals(),
|
|
568
|
+
"import": {
|
|
569
|
+
"format": FORMAT,
|
|
570
|
+
"conventions": CONVENTIONS,
|
|
571
|
+
"trace_id": trace_id,
|
|
572
|
+
"files": list(files),
|
|
573
|
+
"notes": notes,
|
|
574
|
+
},
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def model_of(document: dict) -> str:
|
|
579
|
+
"""The first LLM span's model name, or "" — what the importer shows."""
|
|
580
|
+
for raw in trace.raw_spans_of(document.get("spans") or []):
|
|
581
|
+
if raw.get("kind") != "llm_request":
|
|
582
|
+
continue
|
|
583
|
+
attrs = raw.get("attributes") or {}
|
|
584
|
+
for key in ("llm.model_name", "langfuse.observation.model.name", "gen_ai.request.model"):
|
|
585
|
+
if isinstance(attrs.get(key), str) and attrs[key].strip():
|
|
586
|
+
return attrs[key].strip()
|
|
587
|
+
return ""
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def read(paths: list, limit: int = DEFAULT_MAX_CONTENT) -> list:
|
|
591
|
+
"""Every trace across *paths*, one document each, in first-seen order.
|
|
592
|
+
|
|
593
|
+
A trace may be split across files (a batch per export), which is why the
|
|
594
|
+
files are read together and grouped afterwards. A file that does not read
|
|
595
|
+
raises :class:`ReadError` — the caller decides whether that is fatal.
|
|
596
|
+
"""
|
|
597
|
+
records = []
|
|
598
|
+
where: dict = {}
|
|
599
|
+
for path in paths:
|
|
600
|
+
for request in read_file(path):
|
|
601
|
+
for rec in flat_spans(request):
|
|
602
|
+
records.append(rec)
|
|
603
|
+
where.setdefault(str(rec["span"].get("traceId") or ""), []).append(
|
|
604
|
+
os.path.basename(path))
|
|
605
|
+
return [
|
|
606
|
+
build(tid, recs, limit, tuple(sorted(set(where.get(tid, ())))))
|
|
607
|
+
for tid, recs in traces_of(records).items()
|
|
608
|
+
]
|