evigauge-trace 0.1.0a25__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. evigauge_trace-0.1.0a25/PKG-INFO +1230 -0
  2. evigauge_trace-0.1.0a25/README.md +1190 -0
  3. evigauge_trace-0.1.0a25/evigauge/__init__.py +0 -0
  4. evigauge_trace-0.1.0a25/evigauge/trace/__init__.py +149 -0
  5. evigauge_trace-0.1.0a25/evigauge/trace/_internal/__init__.py +0 -0
  6. evigauge_trace-0.1.0a25/evigauge/trace/_internal/config.py +36 -0
  7. evigauge_trace-0.1.0a25/evigauge/trace/_internal/queue.py +110 -0
  8. evigauge_trace-0.1.0a25/evigauge/trace/adapters/__init__.py +20 -0
  9. evigauge_trace-0.1.0a25/evigauge/trace/adapters/_fleet.py +307 -0
  10. evigauge_trace-0.1.0a25/evigauge/trace/adapters/crewai.py +364 -0
  11. evigauge_trace-0.1.0a25/evigauge/trace/adapters/langchain.py +324 -0
  12. evigauge_trace-0.1.0a25/evigauge/trace/adapters/microsoft.py +155 -0
  13. evigauge_trace-0.1.0a25/evigauge/trace/adapters/reasonix.py +83 -0
  14. evigauge_trace-0.1.0a25/evigauge/trace/audit/__init__.py +21 -0
  15. evigauge_trace-0.1.0a25/evigauge/trace/audit/checks.py +899 -0
  16. evigauge_trace-0.1.0a25/evigauge/trace/audit/cli.py +156 -0
  17. evigauge_trace-0.1.0a25/evigauge/trace/audit/discovery.py +584 -0
  18. evigauge_trace-0.1.0a25/evigauge/trace/audit/graph.py +192 -0
  19. evigauge_trace-0.1.0a25/evigauge/trace/audit/lockfile.py +230 -0
  20. evigauge_trace-0.1.0a25/evigauge/trace/audit/mapview.py +251 -0
  21. evigauge_trace-0.1.0a25/evigauge/trace/audit/relations.py +211 -0
  22. evigauge_trace-0.1.0a25/evigauge/trace/audit/report.py +219 -0
  23. evigauge_trace-0.1.0a25/evigauge/trace/auto_instrument.py +372 -0
  24. evigauge_trace-0.1.0a25/evigauge/trace/cli.py +38 -0
  25. evigauge_trace-0.1.0a25/evigauge/trace/context_manager.py +230 -0
  26. evigauge_trace-0.1.0a25/evigauge/trace/cost.py +243 -0
  27. evigauge_trace-0.1.0a25/evigauge/trace/decorator.py +90 -0
  28. evigauge_trace-0.1.0a25/evigauge/trace/exporter.py +28 -0
  29. evigauge_trace-0.1.0a25/evigauge/trace/http_exporter.py +139 -0
  30. evigauge_trace-0.1.0a25/evigauge/trace/live/__init__.py +152 -0
  31. evigauge_trace-0.1.0a25/evigauge/trace/live/backend.py +97 -0
  32. evigauge_trace-0.1.0a25/evigauge/trace/live/cost_local.py +47 -0
  33. evigauge_trace-0.1.0a25/evigauge/trace/live/events.py +118 -0
  34. evigauge_trace-0.1.0a25/evigauge/trace/live/render.py +236 -0
  35. evigauge_trace-0.1.0a25/evigauge/trace/schemas.py +98 -0
  36. evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/__init__.py +17 -0
  37. evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/artifacts.py +316 -0
  38. evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/cli.py +240 -0
  39. evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/policy.py +135 -0
  40. evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/report.py +129 -0
  41. evigauge_trace-0.1.0a25/evigauge/trace/wire.py +130 -0
  42. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/PKG-INFO +1230 -0
  43. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/SOURCES.txt +73 -0
  44. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/dependency_links.txt +1 -0
  45. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/entry_points.txt +4 -0
  46. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/requires.txt +23 -0
  47. evigauge_trace-0.1.0a25/evigauge_trace.egg-info/top_level.txt +4 -0
  48. evigauge_trace-0.1.0a25/pxcore/__init__.py +132 -0
  49. evigauge_trace-0.1.0a25/pxcore/agent_tools.py +50 -0
  50. evigauge_trace-0.1.0a25/pxcore/calibration/__init__.py +105 -0
  51. evigauge_trace-0.1.0a25/pxcore/calibration/battery.py +168 -0
  52. evigauge_trace-0.1.0a25/pxcore/calibration/profiles/claude-fable-5.json +22 -0
  53. evigauge_trace-0.1.0a25/pxcore/classifier.py +122 -0
  54. evigauge_trace-0.1.0a25/pxcore/drift.py +59 -0
  55. evigauge_trace-0.1.0a25/pxcore/fidelity.py +35 -0
  56. evigauge_trace-0.1.0a25/pxcore/gate.py +46 -0
  57. evigauge_trace-0.1.0a25/pxcore/integrations.py +172 -0
  58. evigauge_trace-0.1.0a25/pxcore/meter.py +39 -0
  59. evigauge_trace-0.1.0a25/pxcore/ops.py +59 -0
  60. evigauge_trace-0.1.0a25/pxcore/renderer/__init__.py +157 -0
  61. evigauge_trace-0.1.0a25/pxcore/renderer/font.py +131 -0
  62. evigauge_trace-0.1.0a25/pxcore/renderer/png.py +49 -0
  63. evigauge_trace-0.1.0a25/pxcore/types.py +129 -0
  64. evigauge_trace-0.1.0a25/pxcore_mcp/__init__.py +15 -0
  65. evigauge_trace-0.1.0a25/pxcore_mcp/__main__.py +34 -0
  66. evigauge_trace-0.1.0a25/pxcore_mcp/server.py +66 -0
  67. evigauge_trace-0.1.0a25/pxcore_mcp/tools.py +120 -0
  68. evigauge_trace-0.1.0a25/pxcore_mcp/transport.py +71 -0
  69. evigauge_trace-0.1.0a25/pxcore_proxy/__init__.py +12 -0
  70. evigauge_trace-0.1.0a25/pxcore_proxy/__main__.py +24 -0
  71. evigauge_trace-0.1.0a25/pxcore_proxy/proxy.py +115 -0
  72. evigauge_trace-0.1.0a25/pxcore_proxy/rewrite.py +16 -0
  73. evigauge_trace-0.1.0a25/pyproject.toml +253 -0
  74. evigauge_trace-0.1.0a25/setup.cfg +4 -0
  75. evigauge_trace-0.1.0a25/tests/test_wire_error_capture.py +66 -0
@@ -0,0 +1,1230 @@
1
+ Metadata-Version: 2.4
2
+ Name: evigauge-trace
3
+ Version: 0.1.0a25
4
+ Summary: Pluggable reasoning-trace observability for LLM applications
5
+ Author: Evigauge
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/evigauge-org/evigauge-python-sdk
8
+ Project-URL: Repository, https://github.com/evigauge-org/evigauge-python-sdk
9
+ Project-URL: Issues, https://github.com/evigauge-org/evigauge-python-sdk/issues
10
+ Keywords: observability,opentelemetry,llm,reasoning,tracing,otel
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
20
+ Classifier: Topic :: System :: Monitoring
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ Requires-Dist: opentelemetry-api<2.0,>=1.24
24
+ Requires-Dist: opentelemetry-sdk<2.0,>=1.24
25
+ Requires-Dist: opentelemetry-exporter-otlp<2.0,>=1.24
26
+ Requires-Dist: pydantic<3.0,>=2.6
27
+ Requires-Dist: tiktoken<1.0,>=0.6
28
+ Requires-Dist: httpx<1.0,>=0.27
29
+ Provides-Extra: shipcheck
30
+ Requires-Dist: pyyaml<7,>=6; extra == "shipcheck"
31
+ Provides-Extra: live
32
+ Requires-Dist: rich<15,>=13; extra == "live"
33
+ Provides-Extra: openllmetry
34
+ Requires-Dist: traceloop-sdk<1.0,>=0.18; extra == "openllmetry"
35
+ Provides-Extra: litellm
36
+ Requires-Dist: litellm<2.0,>=1.40; extra == "litellm"
37
+ Provides-Extra: adapters-microsoft
38
+ Provides-Extra: adapters-langchain
39
+ Requires-Dist: langchain-core<1.0,>=0.2; extra == "adapters-langchain"
40
+
41
+ # evigauge-trace
42
+
43
+ Pluggable reasoning-trace observability for LLM applications.
44
+
45
+ `evigauge-trace` is the **Python** client SDK for the **Evigauge Observability
46
+ Layer**. It captures *reasoning-altitude* spans — the decisions an agent made,
47
+ the sources it used, what each step cost, and the inputs to a reliability
48
+ score — and ships them to an Evigauge backend over OTLP. Unlike raw LLM logging,
49
+ the unit of observation is the *reasoning step*, not the HTTP call.
50
+
51
+ > **The package now includes more than tracing.** One `pip install evigauge-trace`
52
+ > also gives you three tools that ship with the SDK — all covered below:
53
+ > - **`evigauge shipcheck`** — a zero-LLM pre-merge CI gate that catches prompt /
54
+ > model / cost / cache regressions from the traces you already sent.
55
+ > - **`evigauge audit`** — a local, zero-egress security audit + map of an agentic
56
+ > app (MCP servers, agents, tools, hooks), graded against the NSA's MCP
57
+ > security guidance.
58
+ > - **`import pxcore`** (+ `pxcore-mcp` / `pxcore-proxy`) — deterministic
59
+ > image-as-context **token compression**: render dense reference context as
60
+ > images the model reads with native vision, keep exact content as text.
61
+ >
62
+ > See [What's new (0.1.0a13 → a16)](#whats-new-in-010a13--a16) and the
63
+ > [Also in this package](#also-in-this-package--shipcheck-audit-pxcore) section.
64
+
65
+ > **Not a Python shop?** Evigauge's ingest speaks plain OTLP/HTTP, so any
66
+ > language with an OpenTelemetry SDK works today — including Next.js /
67
+ > TypeScript and Vercel AI SDK apps. See [Use from TypeScript /
68
+ > Next.js](#use-from-typescript--nextjs) below. You don't need to spin up
69
+ > Python just to ship spans to us.
70
+
71
+ ## What Evigauge does with your spans (the intelligence layer)
72
+
73
+ `evigauge-trace` is **not a logging service**. It is the client half of an
74
+ *AI-pipeline reconstruction engine*. Once your spans land in our ingest,
75
+ reconstruction engines run server-side on every batch:
76
+
77
+ | Engine | Reads from your spans | Produces |
78
+ |---|---|---|
79
+ | **Decomposition** | parent-child tree + `reasoning_role` | how the query was broken down + decomposition quality score |
80
+ | **Decision Trace** | `evigauge.decision.*` attributes per step | reasoning DAG — which branches were taken vs abandoned, and why |
81
+ | **Sources** | `evigauge.sources.used / consulted / dropped` URL lists | sources matrix + **live Exa-backed web verification** + per-domain authority prior that learns over time per workspace |
82
+ | **Reliability v2** | LLM completions + cited sources | per-claim hallucination score |
83
+ | **Correlation** | model, tokens, cost, pipeline shape | over-provisioning signals — where a premium model ran a simple query, and the measured saving of a cheaper one |
84
+ | **Savings Advisor** | prompt/context token structure | prompt-caching and context-optimization recommendations (advice only; never actuates) |
85
+ | **Alignment** | completions + your workspace knowledge base | whether answers align with your own source-of-truth documents |
86
+ | **External Cross-Check** | extracted claims + live web | independent verification of stage-1 claims |
87
+
88
+ Some engines are per-workspace toggles and some ship dark; ask your workspace owner
89
+ what is enabled.
90
+
91
+ You do **not** call any "compute" endpoint. Spans go in → artifacts appear in
92
+ the Read API seconds later. The semantic conventions on each span are what
93
+ unlock this — a raw "LLM call, 1.2s, 800 tokens" span gives you cost and
94
+ latency only; the rest of the magic needs the attributes documented in
95
+ [Semantic conventions](#semantic-conventions-for-non-python-languages) below.
96
+
97
+ ## Install
98
+
99
+ ```bash
100
+ pip install evigauge-trace
101
+ ```
102
+
103
+ Already have it? **Upgrade to `0.1.0a16` or later** — you get Ship Check, the
104
+ agent audit, and pxcore, and every release before `0.1.0a12` silently discards
105
+ its auto-instrumented spans (see [What's new](#whats-new-in-010a13--a16)):
106
+
107
+ ```bash
108
+ pip install --upgrade --pre evigauge-trace
109
+ ```
110
+
111
+ The `--pre` is required. Every published version is a pre-release
112
+ (`0.1.0a1` … `0.1.0a12`) and `pip` will not upgrade between pre-releases without
113
+ it. To see what you actually have versus what exists:
114
+
115
+ ```bash
116
+ pip show evigauge-trace # installed
117
+ pip index versions --pre evigauge-trace # available
118
+ ```
119
+
120
+ ## What's new in `0.1.0a13` → `a16`
121
+
122
+ The SDK grew three capabilities beyond tracing, all shipped in this one package:
123
+
124
+ - **`0.1.0a13` — Ship Check.** `evigauge shipcheck`: a pre-merge CI gate that makes
125
+ **no LLM call and never re-runs your agent**. It projects a prompt/model/config
126
+ change against the traces you already sent — cost per call / trace / month,
127
+ prompt-cache safety, structure regressions — and a second 100%-local gate flags
128
+ Claude/agent artifacts (`CLAUDE.md`, `.mcp.json`, …) shipping inside your
129
+ production build. Also: prompt versioning + a Prompt Health Score, and the
130
+ `evigauge.prompt_id` / `evigauge.prompt_version` envelope keys for exact baseline
131
+ matching. See [Also in this package](#also-in-this-package--shipcheck-audit-pxcore).
132
+ - **`0.1.0a14` — Agent Security Audit.** `evigauge audit`: **local and zero-egress**
133
+ (no network, no LLM, nothing it finds leaves the machine). It maps an agentic
134
+ app's topology — agents, subagents, MCP servers, tools, hooks — plus a granular
135
+ reachability layer (capabilities, datastores, endpoints, secrets-by-name), and
136
+ audits it against the **NSA MCP Security CSI**: tool-description injection
137
+ (incl. hidden unicode), blanket OAuth scopes, unpinned `npx -y` boot-time code
138
+ execution, tool-name collisions, cleartext credentials, and secret→internet
139
+ exfiltration paths. Its headline is a committed `.evigauge/agentmap.lock` that
140
+ turns a silent capability change ("rug pull") into a reviewable PR diff.
141
+ - **`0.1.0a15`–`a16` — pxcore token compression.** `import pxcore`, plus the
142
+ `pxcore-mcp` and `pxcore-proxy` commands. Deterministic image-as-context
143
+ compression, folded into this distribution (no separate install). See
144
+ [pxcore](#pxcore--token-compression-import-pxcore) below.
145
+ - **`0.1.0a21` — Prompt-injection susceptibility audit.** A new check family in
146
+ `evigauge audit` (and Gate 3 of `evigauge shipcheck`). It reads **every system
147
+ instruction** in the repo — `CLAUDE.md`/`AGENTS.md`/`.cursorrules`, in-code
148
+ `system=`/`SYSTEM_PROMPT` literals, subagents, skills, `prompts:` files — and for
149
+ each gap names the injection **type** and the concrete **mitigation**, cited
150
+ per-check to OWASP LLM01:2025 / NIST AI 100-2e2025 / MITRE ATLAS / NSA-CISA.
151
+ Applicability-gated (a hardened prompt stays silent); a secret in a prompt is a
152
+ provable FAIL (value never emitted). Same **local, zero-egress, zero-LLM** contract;
153
+ the full report is local-only. See
154
+ [Also in this package](#also-in-this-package--shipcheck-audit-pxcore).
155
+
156
+ ## Also in this package — shipcheck, audit, pxcore
157
+
158
+ These ship with `evigauge-trace`; nothing extra to install.
159
+
160
+ ### Ship Check — `evigauge shipcheck` (zero-LLM PR gate)
161
+
162
+ ```bash
163
+ pip install "evigauge-trace[shipcheck]" # the extra is just PyYAML, to read the policy file
164
+ evigauge shipcheck --help
165
+ ```
166
+
167
+ Declare a policy in `.evigauge/shipcheck.yml` (prompt files, cost thresholds), then
168
+ run `evigauge shipcheck` in CI. Exit `0` = pass, `1` = a gate failed the policy,
169
+ `2` = the check could not run (a backend outage is never a silent pass). It talks
170
+ to your workspace with `EVIGAUGE_API_KEY`; the artifact-leak gate is fully local and
171
+ needs no key (`evigauge shipcheck --skip-prompts`). Full guide: `docs/shipcheck.md`.
172
+
173
+ ### Agent Security Audit — `evigauge audit` (local, zero-egress)
174
+
175
+ ```bash
176
+ evigauge audit --map agentmap.html # writes a self-contained, offline HTML map
177
+ evigauge audit --approve-all # bless the current agent map as the baseline
178
+ evigauge audit --injection-report pi.md # prompt-injection susceptibility report (local only)
179
+ ```
180
+
181
+ Runs automatically as a third gate of `evigauge shipcheck` too (WARN by default).
182
+ No API key, no network. Full spec: `docs/agent-map-security-audit.md`.
183
+
184
+ **Prompt-injection susceptibility (a21).** The same audit also checks whether each
185
+ **system instruction** is defenseless against injection arriving at runtime — and
186
+ names the injection **type** and the exact **mitigation** for every gap. It reads
187
+ `CLAUDE.md`/`AGENTS.md`/`.cursorrules`, in-code `system=`/`SYSTEM_PROMPT` literals,
188
+ subagents, skills, and your `prompts:` files (all read locally, sent nowhere).
189
+ Checks are **applicability-gated** so a hardened prompt stays silent; a secret in a
190
+ prompt is a provable FAIL (shape + location, never the value). Each finding cites
191
+ its own source (OWASP LLM01:2025, NIST AI 100-2e2025, MITRE ATLAS AML.T0051,
192
+ NSA/CISA). Configure it under `audit.injection` in `.evigauge/shipcheck.yml`:
193
+
194
+ ```yaml
195
+ audit:
196
+ injection:
197
+ mode: warn # warn (default) | fail (exit 1, blocks the push) | off
198
+ report: prompt-injection-audit.md # LOCAL path for the full report
199
+ ```
200
+
201
+ The full report is an attacker's roadmap, so it is written **locally only**; the
202
+ shared PR comment gets the verdict + finding categories, never the evidence.
203
+
204
+ ### pxcore — token compression (`import pxcore`)
205
+
206
+ Cut input tokens by rendering **dense, reference** context as images the model
207
+ reads with its native vision, while keeping exact content (ids, paths, hashes,
208
+ code you edit) as text. **No LLM in the compression path; nothing leaves the
209
+ machine.** It is **default-off per model** and images only where a calibration
210
+ battery has proven that model reads imaged text back reliably — everything else
211
+ stays text (safe).
212
+
213
+ **Python — library (compress context you assembled, or give your agent tools):**
214
+
215
+ ```python
216
+ import pxcore
217
+ from pxcore.calibration import load_profile
218
+
219
+ profile = load_profile("claude-fable-5")
220
+ blocks = pxcore.to_anthropic(big_reference_text, profile) # image+factsheet, or text
221
+ new_body, saved = pxcore.compress_anthropic(anthropic_request_body) # whole-request
222
+
223
+ from pxcore import agent_tools # ready-to-register agent tools
224
+ result = agent_tools.read("large/log.txt") # returns Anthropic content blocks
225
+ ```
226
+
227
+ **TypeScript / Next.js — via the MCP server (no npm package needed):**
228
+
229
+ ```bash
230
+ pxcore-mcp --http --port 8765 # your TS code connects with @modelcontextprotocol/sdk
231
+ ```
232
+
233
+ **Claude Code — the in-path proxy (also images system prompt + history):**
234
+
235
+ ```bash
236
+ pxcore-proxy # then launch with ANTHROPIC_BASE_URL=http://127.0.0.1:8788
237
+ ```
238
+
239
+ Two one-time gates before it actually compresses (until then it safely returns
240
+ text): **calibrate your model**, and **confirm image passthrough**. Full guide:
241
+ `docs/frontend-pxcore-guide.md` and `docs/pxcore-readme.md`.
242
+
243
+ ## What's new in `0.1.0a12`
244
+
245
+ **Bug fix — auto-instrumented spans were being discarded.** `init(auto_instrument=True)`
246
+ produced spans that the ingest API rejected and dead-lettered, silently, behind an
247
+ HTTP `202`. Two independent causes, both fixed:
248
+
249
+ 1. **The `evigauge.*` envelope was never stamped.** `@observe` and `ReasoningTrace`
250
+ attached `evigauge.schema_version` / `org_id` / `workspace_id` / `project_id` to
251
+ each span; the auto-instrument wrappers did not. Ingest validates the span's own
252
+ attributes, so every one of them failed as `schema_version_unknown`.
253
+ 2. **Three attributes the envelope schema forbids.** `EvigaugeSpanAttributes` is
254
+ `extra="forbid"`, so a single unknown `evigauge.*` key rejects the *entire* span.
255
+ `evigauge.cost.unpriced_model`, `evigauge.instrumentation_error` (auto-instrument) and
256
+ `evigauge.domain` (`ReasoningTrace(domain=…)`) are now emitted under the
257
+ `evigauge_sdk.` prefix, which ingest ignores rather than validates.
258
+
259
+ > `ReasoningTrace(domain=…)` no longer destroys the span. It is still **not** a
260
+ > queryable dimension server-side — Evigauge does not index `domain`.
261
+
262
+ **New — `transport="direct"`.** Export spans straight to `backend_url` over HTTPS. No
263
+ OpenTelemetry collector, no gRPC:
264
+
265
+ ```python
266
+ init(..., transport="direct") # or set EVIGAUGE_TRANSPORT=direct
267
+ ```
268
+
269
+ The **default is unchanged** (`transport="collector"`, OTLP/gRPC to
270
+ `collector_endpoint`), so upgrading does not alter the behaviour of an existing
271
+ deployment. If you keep the collector, set `encoding: json` on its `otlphttp`
272
+ exporter — Evigauge's ingest answers OTLP/**protobuf** with a `415`.
273
+
274
+ **Fixed — `init()` raised on Windows.** The WAL was opened for append before the
275
+ previous run's WAL was replayed, and renaming an open file raises `PermissionError`
276
+ on Windows (it silently succeeds on POSIX). Replay now happens first.
277
+
278
+ **Fixed — a cold-connection timeout dropped a span batch.** `BatchSpanProcessor` does
279
+ not retry a failed export. The direct exporter now retries once on transport errors.
280
+
281
+ ## Quickstart
282
+
283
+ Call `init()` once at process startup:
284
+
285
+ ```python
286
+ from evigauge.trace import init
287
+
288
+ init(
289
+ org_id="pwc",
290
+ workspace_id="chatpwc",
291
+ project_id="due-diligence",
292
+ backend_url="https://ingest.evigauge.ai",
293
+ api_key="opx_live_...", # your workspace API key
294
+ transport="direct", # no collector to run
295
+ )
296
+ ```
297
+
298
+ With `auto_instrument=True` (the default), every LLM call made through
299
+ litellm / anthropic / openai is now traced. No other code changes required.
300
+
301
+ ## The three integration patterns
302
+
303
+ ### Pattern A — Auto-instrument (zero code changes)
304
+
305
+ `init(auto_instrument=True)` monkey-patches the litellm / anthropic / openai
306
+ client methods at startup. Every completion call emits a `gen_ai.*` span
307
+ carrying the model, token usage, and computed USD cost. This is the default;
308
+ you get it just by calling `init()`.
309
+
310
+ ### Pattern B — the `@observe` decorator
311
+
312
+ Wrap any function — sync or async — to emit a reasoning span for it:
313
+
314
+ ```python
315
+ from evigauge.trace import observe
316
+
317
+ @observe(reasoning_role="decomposer", node_type="decomposer", name="plan.decompose")
318
+ async def decompose(query: str) -> list[str]:
319
+ ...
320
+ ```
321
+
322
+ Nested `@observe` calls auto-parent via OpenTelemetry context — a decorated
323
+ function called inside another decorated function becomes its child span, so
324
+ the reasoning tree falls out of normal call structure.
325
+
326
+ ### Pattern C — the `ReasoningTrace` context manager
327
+
328
+ For explicit, structured traces — when you want to record decisions, sources,
329
+ and costs by hand:
330
+
331
+ ```python
332
+ from evigauge.trace import ReasoningTrace
333
+
334
+ with ReasoningTrace(name="answer.query", reasoning_role="synthesis") as trace:
335
+ trace.record_cost(model="claude-opus-4-6", input_tokens=1200, output_tokens=400)
336
+ trace.record_sources(used=[...], consulted=[...], dropped=[...])
337
+ trace.record_decision(selected="opt_a", rules_fired=[...], scores={...})
338
+ with trace.subnode(name="retrieve", node_type="retriever") as node:
339
+ node.record_sources(...)
340
+ ```
341
+
342
+ ### Framework adapters
343
+
344
+ For agent frameworks, one line instruments a whole crew:
345
+
346
+ ```python
347
+ from evigauge.trace.adapters.microsoft import instrument_microsoft_agents
348
+ instrument_microsoft_agents(crew) # Microsoft Agent Framework
349
+
350
+ from evigauge.trace.adapters.reasonix import instrument_reasonix
351
+ instrument_reasonix(orchestrator) # Reasonix orchestrator
352
+ ```
353
+
354
+ **CrewAI** — one call instruments the whole crew (root + one span per agent, with
355
+ tool calls nested under the agent that made them):
356
+
357
+ ```python
358
+ import os
359
+ os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" # BEFORE importing crewai (see note)
360
+ from evigauge.trace import init
361
+ init(org_id=..., workspace_id=..., project_id=..., backend_url=..., api_key=...,
362
+ transport="direct", auto_instrument=False) # leave auto_instrument OFF for CrewAI
363
+
364
+ from crewai import Crew
365
+ from evigauge.trace.adapters import instrument_crewai
366
+ crew = Crew(agents=[...], tasks=[...])
367
+ instrument_crewai(crew) # then kick off as usual
368
+ crew.kickoff(inputs={...})
369
+ ```
370
+
371
+ > **You must set `CREWAI_DISABLE_TELEMETRY=true` before importing crewai.** CrewAI
372
+ > registers its own global OpenTelemetry provider; OTel allows exactly one, so if
373
+ > CrewAI's wins, evigauge's exporter is never attached and spans are dropped. The
374
+ > adapter warns you loudly if it detects this, but it can't undo it. Also keep
375
+ > `auto_instrument=False` — CrewAI's per-call LLM spans otherwise mis-name the
376
+ > fleet; the adapter puts total token usage on the root from `crew.usage_metrics`.
377
+
378
+ **LangChain** — a callback handler; pass it per run or register it globally:
379
+
380
+ ```python
381
+ from evigauge.trace.adapters import get_langchain_handler, instrument_langchain
382
+
383
+ handler = get_langchain_handler()
384
+ chain.invoke(inputs, config={"callbacks": [handler]}) # per-run
385
+
386
+ instrument_langchain() # or: best-effort global registration for every run
387
+ ```
388
+
389
+ Token usage comes from LangChain's `on_llm_end`, so leave `auto_instrument=False`
390
+ here too. Retriever calls emit a `retrieval` node with `evigauge.sources` populated.
391
+
392
+ Adapters are duck-typed — the CrewAI one imports nothing from crewai, and the
393
+ LangChain one imports its base class lazily, so `import evigauge.trace.adapters`
394
+ never fails for a project that doesn't use them, and no version is pinned.
395
+
396
+ ## Per-employee attribution — `evigauge.end_user`
397
+
398
+ When your app serves many employees, tag each trace with the actor who made
399
+ the request. The alignment engine groups its on-/off-mission verdicts by this
400
+ key (and buckets unattributed traces under `(unattributed)`). It's optional and
401
+ per-request — set it wherever you already set `reasoning_role`:
402
+
403
+ ```python
404
+ # On the decorator:
405
+ @observe(reasoning_role="synthesis", end_user="alice@acme.com")
406
+ def handle_query(...): ...
407
+
408
+ # On a ReasoningTrace / subnode:
409
+ with ReasoningTrace(name="chat.turn", end_user="alice@acme.com") as trace:
410
+ ...
411
+
412
+ # Or imperatively, once the actor is resolved mid-request:
413
+ with ReasoningTrace(name="chat.turn") as trace:
414
+ trace.record_end_user(current_user.email) # parity with TS setEvigaugeEndUser
415
+ ```
416
+
417
+ Distinct from the generic `evigauge.user_id` envelope tag — `end_user` is the
418
+ dedicated actor key the alignment feature reads. An empty/omitted value is a
419
+ no-op (the trace stays unattributed). (TypeScript: use `setEvigaugeEndUser(span,
420
+ endUser)` — see the instrumentation snippet below.)
421
+
422
+ ## Agent Fleet Monitor — real-time topology & flags
423
+
424
+ The **Agent Fleet Monitor** is a *separate, real-time* capability from the batch
425
+ reconstruction engines above. It folds your live span stream into **one
426
+ persistent topology map per fleet** — nodes are your agents, edges are the
427
+ hand-offs between them — and shows **data-volume in/out** (tokens + cache
428
+ tokens) per node and edge, flagging **latency spikes** and **errors** live on
429
+ the dashboard. It is deterministic and **LLM-free**: it never reads your
430
+ prompt/response *content*, only counts and status.
431
+
432
+ **It works with zero changes** — it auto-discovers agents from the spans you
433
+ already emit. But because it identifies a node by this ladder…
434
+
435
+ ```
436
+ gen_ai.agent.name → gen_ai.system | evigauge.reasoning_role | evigauge.node_type → (unattributed)
437
+ ```
438
+
439
+ …if you don't name your agents, distinct agents that share a role collapse into
440
+ one coarse node (e.g. a fleet shows up as `retrieval|retrieval` instead of
441
+ `Planner`, `Researcher`, `Writer`). **To get a clean, correct topology, set a
442
+ stable `gen_ai.agent.name` on each agent's span.** That single attribute is the
443
+ whole ask.
444
+
445
+ ### What the monitor reads (set these; most are already emitted)
446
+
447
+ | Signal | Attribute(s) | Who sets it | Effect |
448
+ |---|---|---|---|
449
+ | **Node identity** | `gen_ai.agent.name` (fallbacks: `agent.name`, `gen_ai.agent.id`, `agent.id`) | **you** (explicit) | The node label. Keep it **stable** across runs. |
450
+ | **Edge (A → B hand-off)** | `parent_span_id` | OTel automatically | Every hand-off = one edge. Nest your spans (see below). |
451
+ | **Volume in/out** | `gen_ai.usage.input_tokens` / `output_tokens` | auto-instrument / your LLM span | tokens-in / tokens-out on the node + edge |
452
+ | **Cache volume** | `gen_ai.usage.cache_read_input_tokens` / `cache_creation_input_tokens` (OpenAI `cached_tokens` also read) | auto-instrument | cache-in / cache-out |
453
+ | **Error flag** | span **status = ERROR** | your `try/except` → record on span | node/edge lights red |
454
+ | **Latency flag** | span duration | OTel automatically | amber flag when a node runs slow vs its own learned baseline |
455
+
456
+ > Fleet grouping in v1 is automatic: each run is grouped by its **entry
457
+ > (first-seen) agent**, so naming the *first* agent in a run especially matters.
458
+
459
+ ### The one rule that determines the map: **name the span that carries the tokens**
460
+
461
+ The monitor folds **every span** into a node keyed by its identity, and a node's
462
+ **token/cache volume is the usage on the spans that share that identity.** The
463
+ in/out token counts live on your **LLM-call spans** (auto-instrument /
464
+ `traceLlmCall` / Vercel AI SDK) — *not* on a wrapper span. So:
465
+
466
+ - Set `gen_ai.agent.name` **on the LLM-call span itself** → that call's volume
467
+ folds onto the agent's node. This is what you want.
468
+ - Leave LLM-call spans unnamed → they group into a node named after the
469
+ provider (`anthropic`, `openai`) via the `gen_ai.system` fallback, and the
470
+ volume lands *there*. Still a valid view — just per-provider, not per-agent.
471
+ - Edges come from `parent_span_id`: nest spans so agent B's work runs inside
472
+ agent A's span, and you get the A → B hand-off edge for free.
473
+
474
+ ### Python
475
+
476
+ **Recommended — keep auto-instrument on (zero per-call code).** Auto-instrument
477
+ already emits one span per LLM call carrying the token/cache volume. Name your
478
+ *agents* with `@observe` (or `ReasoningTrace`) and you get the hand-off topology
479
+ for free — nested calls auto-parent into edges:
480
+
481
+ ```python
482
+ from evigauge.trace import init, observe
483
+ from opentelemetry.trace import get_current_span
484
+
485
+ init(org_id="acme", workspace_id="chat", project_id="ui",
486
+ backend_url="https://evigauge.internal.example.com", api_key="opx_live_…")
487
+ # auto_instrument=True is the default -> LLM calls emit their own spans.
488
+
489
+ @observe(reasoning_role="planner", node_type="planner", name="plan.step")
490
+ async def planner(question):
491
+ get_current_span().set_attribute("gen_ai.agent.name", "Planner") # STABLE label
492
+ plan = await ask_llm(f"Plan: {question}")
493
+ return await researcher(plan) # child span -> Planner → Researcher edge
494
+
495
+ @observe(reasoning_role="retriever", node_type="researcher", name="research.step")
496
+ async def researcher(plan):
497
+ get_current_span().set_attribute("gen_ai.agent.name", "Researcher")
498
+ return await ask_llm(plan) # auto-instrumented LLM call nests here
499
+ ```
500
+
501
+ **What you get:** agent nodes `Planner` and `Researcher` with the hand-off edge
502
+ between them, plus the LLM calls as **provider nodes** (`anthropic`, `openai`)
503
+ that carry the token/cache volume, hanging off whichever agent made them. This
504
+ is the low-friction path and it keeps cost + reliability + decomposition
505
+ instrumentation fully intact.
506
+
507
+ **Advanced — per-agent volume (fold the tokens onto the *agent* node).** The
508
+ token counts live on the LLM-call span, so to move volume off the provider node
509
+ and onto the agent node, `gen_ai.agent.name` has to be on *that* span. Since
510
+ auto-instrument owns it and doesn't set an agent name, you hand-instrument the
511
+ call — and turn auto-instrument off (`init(..., auto_instrument=False)`) **or**
512
+ skip it for these calls, so the same tokens aren't also counted on a duplicate
513
+ `anthropic` node:
514
+
515
+ ```python
516
+ from opentelemetry.trace import get_tracer
517
+ tracer = get_tracer("my-ai-app")
518
+
519
+ def call_as_agent(agent_name, role, messages):
520
+ with tracer.start_as_current_span(f"agent.{agent_name}") as span:
521
+ span.set_attribute("gen_ai.agent.name", agent_name) # node identity
522
+ span.set_attribute("evigauge.reasoning_role", role)
523
+ r = anthropic_client.messages.create(
524
+ model="claude-opus-4-6", max_tokens=800, messages=messages)
525
+ span.set_attribute("gen_ai.usage.input_tokens", r.usage.input_tokens)
526
+ span.set_attribute("gen_ai.usage.output_tokens", r.usage.output_tokens)
527
+ return r
528
+ ```
529
+
530
+ > Trade-off: `auto_instrument=False` is **global** — it disables the automatic
531
+ > cost/reliability spans everywhere, not just for the fleet map. Only reach for
532
+ > the advanced path if per-agent *volume* attribution matters more than
533
+ > zero-effort cost tracking. For most teams the recommended path (provider nodes
534
+ > carry volume, agent nodes carry structure) is the right call.
535
+
536
+ Errors flag automatically — a raised exception sets the span status to ERROR,
537
+ which the monitor turns into a red flag. No extra code.
538
+
539
+ ### Next.js / TypeScript
540
+
541
+ Put `gen_ai.agent.name` on the **token-bearing** span. Extend the `traceLlmCall`
542
+ helper from [Pattern B](#pattern-b--raw-anthropic-aisdk--openai-calls-auto-instrument-equivalent)
543
+ with an `agentName` — one line — so identity and volume land on the same node:
544
+
545
+ ```ts
546
+ // lib/evigauge.ts — add agentName to the existing helper
547
+ export async function traceLlmCall<T>(
548
+ fn: () => Promise<T>,
549
+ opts: { system: System; model: string; agentName?: string },
550
+ ): Promise<T> {
551
+ return tracer.startActiveSpan(`gen_ai.${opts.system}.completion`, async (span) => {
552
+ if (opts.agentName) span.setAttribute("gen_ai.agent.name", opts.agentName); // node label + volume
553
+ span.setAttribute("gen_ai.system", opts.system);
554
+ span.setAttribute("gen_ai.request.model", opts.model);
555
+ // ...unchanged: attachUsage() stamps gen_ai.usage.* -> this node's volume,
556
+ // recordException + ERROR status -> red flag, span.end() in finally.
557
+ });
558
+ }
559
+ ```
560
+
561
+ ```ts
562
+ // app/api/answer/route.ts — two-agent fleet: Planner → Researcher
563
+ import { traceLlmCall } from "@/lib/evigauge";
564
+ import Anthropic from "@anthropic-ai/sdk";
565
+
566
+ const anthropic = new Anthropic();
567
+
568
+ export async function POST(req: Request) {
569
+ const { question } = await req.json();
570
+
571
+ // Planner's call is the parent span...
572
+ const plan = await traceLlmCall(
573
+ () => anthropic.messages.create({
574
+ model: "claude-opus-4-6", max_tokens: 200,
575
+ messages: [{ role: "user", content: `Plan: ${question}` }],
576
+ }),
577
+ { system: "anthropic", model: "claude-opus-4-6", agentName: "Planner" },
578
+ );
579
+
580
+ // ...Researcher's call nests inside it -> Planner → Researcher edge.
581
+ const answer = await traceLlmCall(
582
+ () => anthropic.messages.create({
583
+ model: "claude-opus-4-6", max_tokens: 800,
584
+ messages: [{ role: "user", content: String(plan.content) }],
585
+ }),
586
+ { system: "anthropic", model: "claude-opus-4-6", agentName: "Researcher" },
587
+ );
588
+
589
+ return Response.json({ answer: answer.content });
590
+ }
591
+ ```
592
+
593
+ **Vercel AI SDK users:** each model call already emits its own span with the
594
+ token volume — wrap it in a `tracer.startActiveSpan(...)` on which you set
595
+ `gen_ai.agent.name`, so the AI SDK's usage folds onto the named agent node.
596
+
597
+ That's the whole integration: **name the token-bearing spans and keep the calls
598
+ nested.** Edges, volume, latency baselines, and error flags are derived for you.
599
+ Read it back over REST + WebSocket per
600
+ `docs/fleet-monitor-dashboard-integration.md`.
601
+
602
+ ## Use from TypeScript / Next.js
603
+
604
+ `evigauge-trace` is the Python sugar. The transport is plain **OTLP/HTTP**, so
605
+ your Next.js / TypeScript / Vercel-AI-SDK / Cloudflare-Workers app ships to
606
+ the exact same ingest with the exact same backend intelligence — *no Python
607
+ process required*.
608
+
609
+ ### Install (Next.js, Node runtime)
610
+
611
+ ```bash
612
+ npm install \
613
+ @opentelemetry/api \
614
+ @opentelemetry/sdk-node \
615
+ @opentelemetry/exporter-trace-otlp-http \
616
+ @opentelemetry/resources \
617
+ @opentelemetry/semantic-conventions
618
+ ```
619
+
620
+ ### One-time setup — `instrumentation.ts` (Next.js native hook)
621
+
622
+ ```ts
623
+ // instrumentation.ts (project root, sibling of next.config.js)
624
+ import { NodeSDK } from "@opentelemetry/sdk-node";
625
+ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
626
+ import { Resource } from "@opentelemetry/resources";
627
+ import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
628
+
629
+ export async function register() {
630
+ if (process.env.NEXT_RUNTIME !== "nodejs") return;
631
+
632
+ const sdk = new NodeSDK({
633
+ resource: new Resource({
634
+ [SemanticResourceAttributes.SERVICE_NAME]: "my-ai-app",
635
+ "evigauge.ai": process.env.EVIGAUGE_ORG_ID!,
636
+ "evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID!,
637
+ "evigauge.project": process.env.EVIGAUGE_PROJECT_ID!,
638
+ }),
639
+ traceExporter: new OTLPTraceExporter({
640
+ url: "https://ingest.evigauge.ai/v1/traces",
641
+ headers: { "x-evigauge-api-key": process.env.EVIGAUGE_API_KEY! },
642
+ }),
643
+ });
644
+ sdk.start();
645
+ }
646
+ ```
647
+
648
+ Add to `next.config.js`:
649
+
650
+ ```js
651
+ experimental: { instrumentationHook: true }
652
+ ```
653
+
654
+ That's it — your app is now wired to Evigauge.
655
+
656
+ ### Pattern A — Vercel AI SDK (recommended)
657
+
658
+ If you use the `ai` SDK (`generateText`, `streamText`, `generateObject`),
659
+ just enable its native OTel telemetry. Every model call becomes a span
660
+ automatically, and our reconstruction engines pick up cost / latency /
661
+ model / token usage for free:
662
+
663
+ ```ts
664
+ import { generateText } from "ai";
665
+ import { openai } from "@ai-sdk/openai";
666
+
667
+ const result = await generateText({
668
+ model: openai("gpt-4o"),
669
+ prompt: userQuery,
670
+ experimental_telemetry: {
671
+ isEnabled: true,
672
+ functionId: "answer-user",
673
+ metadata: {
674
+ "evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID!,
675
+ "evigauge.trace_kind": "answer",
676
+ },
677
+ },
678
+ });
679
+ ```
680
+
681
+ ### Pattern B — raw `@anthropic-ai/sdk` / `openai` calls (auto-instrument equivalent)
682
+
683
+ Vercel AI SDK is great, but plenty of TS apps call `@anthropic-ai/sdk` or
684
+ `openai` directly. The Python SDK auto-instruments these for you; in TS you
685
+ write a tiny helper *once*, then every LLM call lights up the same way. The
686
+ helper emits the **same `gen_ai.*` semantic conventions** the Python SDK
687
+ emits (mirrors `evigauge_trace/evigauge/trace/auto_instrument.py`), so the
688
+ backend treats both identically.
689
+
690
+ **One-time helper** — `lib/evigauge.ts`:
691
+
692
+ ```ts
693
+ import { trace, SpanStatusCode, Span } from "@opentelemetry/api";
694
+
695
+ type System = "anthropic" | "openai" | "litellm";
696
+
697
+ const tracer = trace.getTracer("evigauge-trace-helpers");
698
+
699
+ export async function traceLlmCall<T>(
700
+ fn: () => Promise<T>,
701
+ opts: { system: System; model: string },
702
+ ): Promise<T> {
703
+ return tracer.startActiveSpan(
704
+ `gen_ai.${opts.system}.completion`,
705
+ async (span: Span) => {
706
+ span.setAttribute("gen_ai.system", opts.system);
707
+ span.setAttribute("gen_ai.request.model", opts.model);
708
+ try {
709
+ const resp = await fn();
710
+ attachUsage(span, resp, opts.system, opts.model);
711
+ span.setStatus({ code: SpanStatusCode.OK });
712
+ return resp;
713
+ } catch (err) {
714
+ span.recordException(err as Error);
715
+ span.setStatus({ code: SpanStatusCode.ERROR });
716
+ throw err;
717
+ } finally {
718
+ span.end();
719
+ }
720
+ },
721
+ );
722
+ }
723
+
724
+ function attachUsage(span: Span, resp: any, system: System, requested: string) {
725
+ try {
726
+ const usage = resp?.usage ?? {};
727
+ const inputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0;
728
+ const outputTokens = usage.output_tokens ?? usage.completion_tokens ?? 0;
729
+ const modelUsed = resp?.model ?? requested;
730
+ const finish =
731
+ resp?.choices?.[0]?.finish_reason ?? // OpenAI / LiteLLM
732
+ resp?.stop_reason ?? // Anthropic
733
+ "";
734
+ span.setAttribute("gen_ai.usage.input_tokens", Number(inputTokens));
735
+ span.setAttribute("gen_ai.usage.output_tokens", Number(outputTokens));
736
+ if (modelUsed) span.setAttribute("gen_ai.response.model", String(modelUsed));
737
+ if (finish) span.setAttribute("gen_ai.response.finish_reason", String(finish));
738
+ } catch (err) {
739
+ span.setAttribute("evigauge.instrumentation_error", String(err));
740
+ }
741
+ }
742
+ ```
743
+
744
+ **Usage with `@anthropic-ai/sdk`** — wrap the call, no other changes:
745
+
746
+ ```ts
747
+ // app/api/answer/route.ts
748
+ import Anthropic from "@anthropic-ai/sdk";
749
+ import { traceLlmCall } from "@/lib/evigauge";
750
+
751
+ const anthropic = new Anthropic();
752
+
753
+ export async function POST(req: Request) {
754
+ const { question } = await req.json();
755
+ const resp = await traceLlmCall(
756
+ () => anthropic.messages.create({
757
+ model: "claude-opus-4-6",
758
+ max_tokens: 100,
759
+ messages: [{ role: "user", content: question }],
760
+ }),
761
+ { system: "anthropic", model: "claude-opus-4-6" },
762
+ );
763
+ return Response.json({ answer: resp.content });
764
+ }
765
+ ```
766
+
767
+ **Usage with `openai`** — identical helper, only `system` changes:
768
+
769
+ ```ts
770
+ import OpenAI from "openai";
771
+ import { traceLlmCall } from "@/lib/evigauge";
772
+
773
+ const openai = new OpenAI();
774
+
775
+ const resp = await traceLlmCall(
776
+ () => openai.chat.completions.create({
777
+ model: "gpt-4o",
778
+ messages: [{ role: "user", content: question }],
779
+ }),
780
+ { system: "openai", model: "gpt-4o" },
781
+ );
782
+ ```
783
+
784
+ What lands in Evigauge — the same span shape the Python SDK's auto-instrument
785
+ produces, so cost/latency/token attribution and the reliability engine work
786
+ out of the box:
787
+
788
+ | Attribute | Value |
789
+ |---|---|
790
+ | `gen_ai.system` | `anthropic` / `openai` / `litellm` |
791
+ | `gen_ai.request.model` | `claude-opus-4-6` |
792
+ | `gen_ai.response.model` | `claude-opus-4-6` (server-confirmed) |
793
+ | `gen_ai.usage.input_tokens` | 12 |
794
+ | `gen_ai.usage.output_tokens` | 8 |
795
+ | `gen_ai.response.finish_reason` | `end_turn` (Anthropic) / `stop` (OpenAI) |
796
+
797
+ Backend cost reconstruction reads `gen_ai.response.model` + the token
798
+ counts — no extra work needed on the client.
799
+
800
+ ### Pattern C — manual reasoning spans (decomposer, decision, sources)
801
+
802
+ To unlock the **Decomposition / Decision Trace / Sources** engines you have
803
+ to set our semantic-convention attributes (see the reference table at the
804
+ bottom). Example — a decomposer step that also records the URLs it consulted
805
+ and the decision it made:
806
+
807
+ ```ts
808
+ import { trace, SpanStatusCode } from "@opentelemetry/api";
809
+
810
+ const tracer = trace.getTracer("my-ai-app");
811
+
812
+ await tracer.startActiveSpan("plan.decompose", async (span) => {
813
+ span.setAttribute("evigauge.reasoning_role", "decomposer");
814
+ span.setAttribute("evigauge.node_type", "decomposer");
815
+
816
+ try {
817
+ const subqueries = await decomposeQuery(userQuery);
818
+ const consulted = await retrieveCandidates(subqueries);
819
+ const cited = pickCitations(consulted);
820
+
821
+ // Sources engine consumes these JSON arrays of URLs:
822
+ span.setAttribute("evigauge.sources.used", JSON.stringify(cited));
823
+ span.setAttribute("evigauge.sources.consulted", JSON.stringify(consulted));
824
+ span.setAttribute("evigauge.sources.dropped", JSON.stringify([]));
825
+
826
+ // Decision Trace engine consumes these:
827
+ span.setAttribute("evigauge.decision.selected", "strategy_a");
828
+ span.setAttribute("evigauge.decision.rules_fired", JSON.stringify(["recency_boost"]));
829
+ span.setAttribute("evigauge.decision.scores", JSON.stringify({ a: 0.82, b: 0.41 }));
830
+
831
+ span.setStatus({ code: SpanStatusCode.OK });
832
+ return subqueries;
833
+ } catch (err) {
834
+ span.recordException(err as Error);
835
+ span.setStatus({ code: SpanStatusCode.ERROR });
836
+ throw err;
837
+ } finally {
838
+ span.end();
839
+ }
840
+ });
841
+ ```
842
+
843
+ Nested `startActiveSpan` calls auto-parent the same way they do in Python —
844
+ the reasoning tree falls out of normal control flow.
845
+
846
+ ### Edge runtime / Cloudflare Workers
847
+
848
+ Full OTel SDKs don't fit in Edge / Workers. Use the `fetch`-based exporter or
849
+ fall back to a `POST` to `/v1/traces` on a Node API route. (Edge-native SDK
850
+ is on the roadmap; ask if you need it sooner.)
851
+
852
+ ## Multi-provider chat apps (UI dropdown picks the model)
853
+
854
+ Most production chat apps let the end-user pick a model from a dropdown —
855
+ Claude, GPT-4o, Gemini, etc. — at runtime. `evigauge-trace` handles this
856
+ without any extra config: **the patches are at the SDK-method level, not at
857
+ some global "current model" setting**. Whichever SDK your dispatcher
858
+ actually calls for a given turn is what gets tagged. Each turn → its own
859
+ span → correctly attributed by provider and model.
860
+
861
+ ```
862
+ UI dropdown → your dispatcher → SDK A.method() ──┐
863
+ ↘ SDK B.method() ──┼──► evigauge-trace tags
864
+ ↘ SDK C.method() ──┘ each one individually
865
+ ```
866
+
867
+ A single chat session with mixed turns shows up as N spans, one per call,
868
+ with `gen_ai.system` and `gen_ai.request.model` set per-turn. Cost rolls up
869
+ per provider, per model — no extra plumbing on your side.
870
+
871
+ ### Python — explicit dispatcher
872
+
873
+ ```python
874
+ from anthropic import Anthropic
875
+ from openai import OpenAI
876
+ from evigauge.trace import init
877
+
878
+ init(org_id="acme", workspace_id="chat", project_id="ui",
879
+ backend_url="…", api_key="opx_live_…")
880
+
881
+ anthropic_client = Anthropic()
882
+ openai_client = OpenAI()
883
+
884
+ def chat(user_selected_model: str, messages: list[dict]) -> str:
885
+ if user_selected_model.startswith("claude"):
886
+ r = anthropic_client.messages.create(
887
+ model=user_selected_model, max_tokens=1000, messages=messages)
888
+ return r.content[0].text
889
+ if user_selected_model.startswith("gpt"):
890
+ r = openai_client.chat.completions.create(
891
+ model=user_selected_model, messages=messages)
892
+ return r.choices[0].message.content
893
+ raise ValueError(f"unknown model: {user_selected_model}")
894
+ ```
895
+
896
+ Each branch hits a patched method; spans land with the right `gen_ai.system`
897
+ (`anthropic` or `openai`) and `gen_ai.request.model`.
898
+
899
+ ### Python — single-call dispatcher via `litellm` (recommended)
900
+
901
+ The cleanest pattern when you support many providers — one function, one
902
+ patched call, ~100 providers covered out of the box:
903
+
904
+ ```python
905
+ import litellm
906
+
907
+ def chat(user_selected_model: str, messages: list[dict]) -> str:
908
+ r = litellm.completion(
909
+ model=user_selected_model, # e.g. "anthropic/claude-opus-4-6", "openai/gpt-4o", "gemini/gemini-pro"
910
+ messages=messages,
911
+ )
912
+ return r.choices[0].message.content
913
+ ```
914
+
915
+ `evigauge-trace` patches `litellm.completion`, so every dropdown selection
916
+ routes through the same patched function. Spans get `gen_ai.system="litellm"`
917
+ and `gen_ai.request.model` carries the actual provider+model
918
+ (`anthropic/claude-opus-4-6`). Our cost engine handles the provider prefix.
919
+
920
+ ### TypeScript — explicit dispatcher with `traceLlmCall`
921
+
922
+ The `traceLlmCall` helper from [Pattern B](#pattern-b--raw-anthropic-aisdk--openai-calls-auto-instrument-equivalent)
923
+ takes `{ system, model }` per call, so the dispatcher writes the right tag
924
+ per branch:
925
+
926
+ ```ts
927
+ import Anthropic from "@anthropic-ai/sdk";
928
+ import OpenAI from "openai";
929
+ import { traceLlmCall } from "@/lib/evigauge";
930
+
931
+ const anthropic = new Anthropic();
932
+ const openai = new OpenAI();
933
+
934
+ export async function chat(selectedModel: string, messages: any[]) {
935
+ if (selectedModel.startsWith("claude")) {
936
+ return traceLlmCall(
937
+ () => anthropic.messages.create({ model: selectedModel, max_tokens: 1000, messages }),
938
+ { system: "anthropic", model: selectedModel },
939
+ );
940
+ }
941
+ if (selectedModel.startsWith("gpt")) {
942
+ return traceLlmCall(
943
+ () => openai.chat.completions.create({ model: selectedModel, messages }),
944
+ { system: "openai", model: selectedModel },
945
+ );
946
+ }
947
+ throw new Error(`unknown model: ${selectedModel}`);
948
+ }
949
+ ```
950
+
951
+ ### TypeScript — Vercel AI SDK provider lookup (recommended)
952
+
953
+ Each `@ai-sdk/*` provider package emits its own OTel span with `gen_ai.system`
954
+ set correctly, so a provider lookup table gives you correct attribution
955
+ across N providers with zero per-provider code:
956
+
957
+ ```ts
958
+ import { generateText } from "ai";
959
+ import { anthropic } from "@ai-sdk/anthropic";
960
+ import { openai } from "@ai-sdk/openai";
961
+ import { google } from "@ai-sdk/google";
962
+
963
+ const providers = { anthropic, openai, google } as const;
964
+ type Provider = keyof typeof providers;
965
+
966
+ export async function chat(provider: Provider, model: string, prompt: string) {
967
+ return generateText({
968
+ model: providers[provider](model),
969
+ prompt,
970
+ experimental_telemetry: {
971
+ isEnabled: true,
972
+ metadata: { "evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID! },
973
+ },
974
+ });
975
+ }
976
+ ```
977
+
978
+ ### Bonus — capturing the dropdown choice itself
979
+
980
+ Useful for debugging "I picked Claude but the answer feels like GPT-4o" —
981
+ stamp the user's intent on a parent span so you can see both layers in the
982
+ trace tree:
983
+
984
+ ```python
985
+ from evigauge.trace import ReasoningTrace
986
+
987
+ with ReasoningTrace(name="chat.turn", reasoning_role="synthesis") as trace:
988
+ trace._span.set_attribute("evigauge.user_selected_model", user_selected_model)
989
+ answer = chat(user_selected_model, messages)
990
+ ```
991
+
992
+ The parent span carries what the user asked for; the child span carries
993
+ what was actually sent. If they diverge, your dispatcher has a bug.
994
+
995
+ ## Cost for any model, any provider
996
+
997
+ `evigauge.cost.usd` is computed for **any model from any provider** — there is no
998
+ fixed allow-list. The SDK resolves a price through a layered strategy, first hit
999
+ wins:
1000
+
1001
+ 1. **Provider-reported cost** — if the response already carries a cost (e.g.
1002
+ OpenRouter usage accounting, or litellm's `response_cost`), it is used
1003
+ as-is. Zero maintenance, exact for every model the gateway prices.
1004
+ 2. **Bundled snapshot** — a version-pinned table of common models (GPT-4o,
1005
+ Claude, Kimi K2, Qwen2.5-72B, GLM-4.6, DeepSeek-V3, …), matched after
1006
+ normalizing the model id, so provider prefixes and variant suffixes resolve:
1007
+ `openrouter/z-ai/glm-4.6:free` → `glm-4.6`.
1008
+ 3. **litellm price map** — if `litellm` is importable, its continuously-updated
1009
+ map (thousands of models across ~100 providers) is consulted. No network.
1010
+ 4. **OpenRouter live** — last resort only: prices are fetched from OpenRouter's
1011
+ `/api/v1/models` and cached (6h). Disable with
1012
+ `EVIGAUGE_PRICING_DISABLE_NETWORK=1`.
1013
+ 5. If none match, cost is `0.0` **and** `evigauge.cost.unpriced_model` is set — a
1014
+ missing price is always flagged, never a silent zero.
1015
+
1016
+ `evigauge.cost.model_pricing_version` records which source produced the figure
1017
+ (`provider`, the snapshot version, `litellm:<ver>`, or `openrouter-live`), so
1018
+ the cost is reproducible server-side. This works identically through OpenRouter
1019
+ or any gateway — pass the model as usual
1020
+ (`litellm.completion(model="openrouter/moonshotai/kimi-k2", …)`) and the span
1021
+ carries an accurate `evigauge.cost.usd`.
1022
+
1023
+ ## Reading the computed artifacts — Read API
1024
+
1025
+ Once spans are in, your Next.js dashboard / debug UI / analytics can fetch
1026
+ the reconstructed intelligence over plain REST. All endpoints are at
1027
+ `https://api.evigauge.ai/v1/observ/*` and authenticated with the same
1028
+ `x-evigauge-api-key` + `x-evigauge-workspace-id` headers.
1029
+
1030
+ | Endpoint | What you get back |
1031
+ |---|---|
1032
+ | `GET /v1/observ/traces?workspace_id=…` | paginated trace list |
1033
+ | `GET /v1/observ/traces/{tid}` | full trace summary |
1034
+ | `GET /v1/observ/traces/{tid}/decomposition` | reconstructed decomposition tree |
1035
+ | `GET /v1/observ/traces/{tid}/decision-trace` | reasoning DAG |
1036
+ | `GET /v1/observ/traces/{tid}/sources` | sources matrix (cited / dropped / recommended-unseen, each with a live web-verification verdict) |
1037
+ | `GET /v1/observ/traces/{tid}/reliability` | per-claim hallucination score |
1038
+ | `GET /v1/observ/usage?workspace_id=…` | cost + token usage rollups |
1039
+ | `GET /v1/observ/drift?workspace_id=…` | daily drift signal |
1040
+
1041
+ Quick Next.js example — a server component showing the sources matrix:
1042
+
1043
+ ```ts
1044
+ // app/traces/[tid]/sources/page.tsx
1045
+ export default async function Page({ params }: { params: { tid: string } }) {
1046
+ const res = await fetch(
1047
+ `https://api.evigauge.ai/v1/observ/traces/${params.tid}/sources`,
1048
+ {
1049
+ headers: {
1050
+ "x-evigauge-api-key": process.env.EVIGAUGE_API_KEY!,
1051
+ "x-evigauge-workspace-id": process.env.EVIGAUGE_WORKSPACE_ID!,
1052
+ },
1053
+ cache: "no-store",
1054
+ },
1055
+ );
1056
+ const sources = await res.json();
1057
+ return <SourcesMatrix data={sources} />;
1058
+ }
1059
+ ```
1060
+
1061
+ The OpenAPI / Swagger UI for every endpoint lives at
1062
+ `https://api.evigauge.ai/v1/observ/docs` (and the ingest at
1063
+ `https://ingest.evigauge.ai/docs`) — fully brand-styled, "Try it out" works
1064
+ in-browser.
1065
+
1066
+ ## Beyond your app — Claude Code enterprise telemetry
1067
+
1068
+ Evigauge also ingests telemetry from **Claude Code** running across your
1069
+ organization — **no `evigauge-trace` install required**. Claude Code emits
1070
+ native OpenTelemetry metrics + events; an IT admin points every install at
1071
+ Evigauge with a single managed-settings file (distributed via MDM / Jamf /
1072
+ Intune / Group Policy) plus a workspace-scoped `cck_live_…` key. You then get,
1073
+ per employee:
1074
+
1075
+ - session count, active time, cost (USD), input / output / cache tokens
1076
+ - **git commits authored in Claude Code** — SHA, message, files changed,
1077
+ ± lines (captured by a small `PostToolUse` hook)
1078
+ - lines of code added / removed, edit accept/reject decisions, top files edited
1079
+
1080
+ This is a *sibling* capability to the SDK: the SDK traces **your LLM app's**
1081
+ reasoning, while Claude Code telemetry tracks **your team's** use of the
1082
+ Claude Code CLI. Both land in the same workspace and share one Read API +
1083
+ Swagger. Compliance is inherited from the same layer — per-workspace
1084
+ `capture_text` opt-in gates prompt/tool-content, every row is PII-redacted at
1085
+ ingest, and Read API access is RBAC-gated + audit-logged.
1086
+
1087
+ **Read API** (`workspace_viewer` role — Better-Auth session or workspace key):
1088
+
1089
+ | Endpoint | What you get back |
1090
+ |---|---|
1091
+ | `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/usage?period=day\|week\|month` | per-employee rollup: tokens / cost / commits / PRs / lines / sessions / active-time / edit decisions |
1092
+ | `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/commits?limit=50` | recent Claude Code-authored commits (SHA, message, files changed, ± lines) |
1093
+ | `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/top-files?period=week` | most-edited files across the workspace |
1094
+
1095
+ Full setup runbook — managed-settings template, the commit hook (`.sh` +
1096
+ `.ps1`), per-platform MDM rollout, and the PII / compliance posture — lives at
1097
+ `docs/claude-code-enterprise-setup.md` in the Evigauge repo.
1098
+
1099
+ ## Semantic conventions (for non-Python languages)
1100
+
1101
+ Set these on your spans (Python SDK does it for you). The reconstruction
1102
+ engines key off these names.
1103
+
1104
+ | Attribute | Type | Purpose |
1105
+ |---|---|---|
1106
+ | `evigauge.ai` / `evigauge.workspace` / `evigauge.project` | resource attrs | tenancy stamp on every span |
1107
+ | `evigauge.reasoning_role` | string | `decomposer` / `retriever` / `synthesizer` / `decision` / `guardrail` |
1108
+ | `evigauge.node_type` | string | finer-grained role; free-form taxonomy |
1109
+ | `gen_ai.agent.name` | string | **Agent Fleet Monitor** node identity — the agent's stable display name (fallbacks: `agent.name` / `gen_ai.agent.id` / `agent.id`). See [Agent Fleet Monitor](#agent-fleet-monitor--real-time-topology--flags). |
1110
+ | `evigauge.sources.used` | JSON array of strings | URLs the step actually cited |
1111
+ | `evigauge.sources.consulted` | JSON array of strings | URLs the step looked at |
1112
+ | `evigauge.sources.dropped` | JSON array of strings | URLs the step considered then rejected |
1113
+ | `evigauge.decision.selected` | string | which option was chosen |
1114
+ | `evigauge.decision.rules_fired` | JSON array of strings | named rules that influenced the decision |
1115
+ | `evigauge.decision.scores` | JSON object `{option: score}` | per-option scoring |
1116
+ | `evigauge.cost.usd` | float | computed USD cost for **any** model (see [Cost for any model](#cost-for-any-model-any-provider)); optional — we derive from tokens if absent |
1117
+ | `evigauge.cost.model_pricing_version` | string | which price source produced the cost (`provider` / snapshot version / `litellm:<ver>` / `openrouter-live`) |
1118
+ | `evigauge.cost.unpriced_model` | string | set only when no price source matched — the cost is `0.0` and this flags it (never a silent zero) |
1119
+ | `gen_ai.system` / `gen_ai.request.model` / `gen_ai.usage.input_tokens` / `gen_ai.usage.output_tokens` | OTel GenAI conventions | LLM call metadata |
1120
+
1121
+ JSON-encode arrays/objects as strings — OTel attributes don't support nested
1122
+ types natively. The backend parses them on the way in.
1123
+
1124
+ ## Configuration — `init()` parameters
1125
+
1126
+ | Parameter | Default | Meaning |
1127
+ |-----------|---------|---------|
1128
+ | `org_id` / `workspace_id` / `project_id` | required | Tenancy identity stamped on every span. |
1129
+ | `backend_url` | required | The Evigauge backend base URL. |
1130
+ | `api_key` | required | Workspace API key (`opx_live_…` / `opx_test_…`). |
1131
+ | `transport` | `"collector"` | `"collector"` = OTLP/gRPC to `collector_endpoint`. `"direct"` = HTTPS POST straight to `backend_url`, no collector, no gRPC. Also settable via `EVIGAUGE_TRANSPORT`; the argument wins. |
1132
+ | `collector_endpoint` | `http://localhost:4317` | OTLP collector endpoint. Ignored when `transport="direct"`. |
1133
+ | `wal_path` | `.evigauge-wal/spans.jsonl` | Write-ahead-log path (see Durability). |
1134
+ | `auto_instrument` | `True` | Patch litellm/anthropic/openai at startup. |
1135
+ | `fail_open` | `False` | See "fail_open scope" in the FAQ. |
1136
+ | `sampler_rate` | `1.0` | Fraction of traces sampled. |
1137
+
1138
+ ## Durability — the write-ahead log
1139
+
1140
+ Every span is `fsync`-ed to a local WAL (`wal_path`) *before* it is handed to the
1141
+ exporter, so a process that crashes mid-export leaves a durable on-disk record of
1142
+ what was in flight. On a clean shutdown the WAL is deleted; spans that never made it
1143
+ out survive on disk.
1144
+
1145
+ **What the next `init()` does with that WAL, precisely:** it counts the entries, logs
1146
+ a warning, and renames the file to `<wal_path>.replayed-<timestamp>`. It does **not**
1147
+ re-export them — full OTLP round-trip replay is not implemented yet. The renamed file
1148
+ is there so an operator can re-emit it manually. Do not read "WAL-backed" as
1149
+ "guaranteed at-least-once delivery to the backend"; read it as "a crash never
1150
+ silently loses the evidence."
1151
+
1152
+ Past `max_wal_bytes` (10 MB default) the WAL write is refused and a counter advances.
1153
+ The span is still exported — only its crash-durability is lost.
1154
+
1155
+ ## Troubleshooting
1156
+
1157
+ - **Spans are accepted but never appear.** The ingest API returns `202` even when it
1158
+ dead-letters every span in the batch — a rejected span is quarantined, not dropped,
1159
+ and the `202` refers to receipt, not validity. On `transport="direct"` the SDK reads
1160
+ the response's `partialSuccess` field and logs
1161
+ `N span(s) were DEAD-LETTERED by ingest`. **If you are on `0.1.0a11` or earlier,
1162
+ this is almost certainly you** — upgrade (see [What's new](#whats-new-in-010a12)).
1163
+ - **`415 OTLP/protobuf is not supported`.** Your collector is exporting protobuf. Set
1164
+ `encoding: json` on its `otlphttp` exporter. The SDK's `transport="direct"` path is
1165
+ unaffected.
1166
+ - **`404` on `/v1/traces`.** You are pointed at an Evigauge backend that predates native
1167
+ OTLP support. Upgrade the backend, or use `transport="direct"` which posts to
1168
+ `/v1/otlp/traces`.
1169
+ - **No spans appear at all.** Check `backend_url` / `collector_endpoint` reachability
1170
+ and that `api_key` is a live (not revoked) workspace key. A `401` is logged
1171
+ explicitly by the direct exporter.
1172
+ - **A custom `evigauge.*` attribute made spans vanish.** `EvigaugeSpanAttributes` is
1173
+ `extra="forbid"`: one unknown `evigauge.`-prefixed key dead-letters the whole span.
1174
+ Put your own attributes under any other prefix — the SDK uses `evigauge_sdk.` for its
1175
+ internal diagnostics for exactly this reason.
1176
+ - **`init() has not been called`.** A `@observe` / `ReasoningTrace` ran before
1177
+ `init()`. Call `init()` once at process startup, before any traced code.
1178
+ - **Auto-instrument patched 0 clients.** None of litellm/anthropic/openai are
1179
+ importable in the process — Pattern A has nothing to patch. Use Pattern B/C.
1180
+ - **`PermissionError` on `init()` (Windows).** Fixed in `0.1.0a12`; earlier releases
1181
+ renamed the WAL file while it was still open.
1182
+
1183
+ ## FAQ
1184
+
1185
+ **Do I need Python to use Evigauge?** No. Ingest accepts standard **OTLP/JSON**, so any
1186
+ language with an OpenTelemetry SDK (TypeScript, Go, Java, .NET, …) can point a stock
1187
+ exporter at it and get the full reconstruction stack. The API key alone identifies
1188
+ your org and workspace, so a stock span needs **no** `evigauge.*` attributes at all:
1189
+
1190
+ ```bash
1191
+ export OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.evigauge.ai
1192
+ export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/json # protobuf gets a 415
1193
+ export OTEL_EXPORTER_OTLP_HEADERS="x-evigauge-api-key=opx_live_…"
1194
+ ```
1195
+
1196
+ Both `/v1/traces` (what an exporter appends to a base endpoint) and
1197
+ `/v1/otlp/traces` work. Add the semantic conventions below to unlock the engines. See
1198
+ [Use from TypeScript / Next.js](#use-from-typescript--nextjs).
1199
+
1200
+ **Then why install this SDK?** For `auto_instrument` (zero-code tracing of
1201
+ litellm/anthropic/openai), the `@observe` / `ReasoningTrace` reasoning-altitude
1202
+ helpers, the cost resolver, and the WAL. For transport alone, a stock OTel SDK is
1203
+ equivalent.
1204
+
1205
+ **Is there a native `@evigauge/trace` npm package?** Not yet — on the roadmap.
1206
+ For now use the raw OpenTelemetry JS SDK with the semantic conventions
1207
+ documented above; Vercel AI SDK users get most of it for free via
1208
+ `experimental_telemetry`.
1209
+
1210
+ **Does `evigauge-trace` import my agent framework?** No. The adapters are
1211
+ duck-typed and import nothing from the framework or from the Evigauge backend.
1212
+
1213
+ **What does `fail_open` actually cover?** `fail_open=True` wraps the **whole**
1214
+ `init()` body — exporter construction, TracerProvider setup, WAL replay, and
1215
+ auto-instrument registration. A bad `collector_endpoint`, a WAL-path I/O error,
1216
+ or an exporter-constructor failure all degrade to a logged exception instead of
1217
+ taking down your startup. `set_config()` runs before the guard, so `get_config()`
1218
+ still works in the degraded state (you simply get no spans). With the default
1219
+ `fail_open=False`, any of those failures raises.
1220
+
1221
+ **Can I emit spans during process shutdown?** No. Do not start new spans after
1222
+ your shutdown hook runs — a span started concurrently with the exporter
1223
+ shutting down can be dropped (this race exists in upstream OpenTelemetry's
1224
+ `BatchSpanProcessor` too). Finish traced work before tearing the process down.
1225
+ (Tracked as deferred note B2-N7.)
1226
+
1227
+ **Is the SDK safe in production?** Yes — span export is buffered and
1228
+ WAL-backed, and a traced function never fails because tracing failed:
1229
+ attribute-extraction errors are caught and recorded on the span, and the
1230
+ original return value passes through untouched.