evigauge-trace 0.1.0a25__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evigauge_trace-0.1.0a25/PKG-INFO +1230 -0
- evigauge_trace-0.1.0a25/README.md +1190 -0
- evigauge_trace-0.1.0a25/evigauge/__init__.py +0 -0
- evigauge_trace-0.1.0a25/evigauge/trace/__init__.py +149 -0
- evigauge_trace-0.1.0a25/evigauge/trace/_internal/__init__.py +0 -0
- evigauge_trace-0.1.0a25/evigauge/trace/_internal/config.py +36 -0
- evigauge_trace-0.1.0a25/evigauge/trace/_internal/queue.py +110 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/__init__.py +20 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/_fleet.py +307 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/crewai.py +364 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/langchain.py +324 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/microsoft.py +155 -0
- evigauge_trace-0.1.0a25/evigauge/trace/adapters/reasonix.py +83 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/__init__.py +21 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/checks.py +899 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/cli.py +156 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/discovery.py +584 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/graph.py +192 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/lockfile.py +230 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/mapview.py +251 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/relations.py +211 -0
- evigauge_trace-0.1.0a25/evigauge/trace/audit/report.py +219 -0
- evigauge_trace-0.1.0a25/evigauge/trace/auto_instrument.py +372 -0
- evigauge_trace-0.1.0a25/evigauge/trace/cli.py +38 -0
- evigauge_trace-0.1.0a25/evigauge/trace/context_manager.py +230 -0
- evigauge_trace-0.1.0a25/evigauge/trace/cost.py +243 -0
- evigauge_trace-0.1.0a25/evigauge/trace/decorator.py +90 -0
- evigauge_trace-0.1.0a25/evigauge/trace/exporter.py +28 -0
- evigauge_trace-0.1.0a25/evigauge/trace/http_exporter.py +139 -0
- evigauge_trace-0.1.0a25/evigauge/trace/live/__init__.py +152 -0
- evigauge_trace-0.1.0a25/evigauge/trace/live/backend.py +97 -0
- evigauge_trace-0.1.0a25/evigauge/trace/live/cost_local.py +47 -0
- evigauge_trace-0.1.0a25/evigauge/trace/live/events.py +118 -0
- evigauge_trace-0.1.0a25/evigauge/trace/live/render.py +236 -0
- evigauge_trace-0.1.0a25/evigauge/trace/schemas.py +98 -0
- evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/__init__.py +17 -0
- evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/artifacts.py +316 -0
- evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/cli.py +240 -0
- evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/policy.py +135 -0
- evigauge_trace-0.1.0a25/evigauge/trace/shipcheck/report.py +129 -0
- evigauge_trace-0.1.0a25/evigauge/trace/wire.py +130 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/PKG-INFO +1230 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/SOURCES.txt +73 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/dependency_links.txt +1 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/entry_points.txt +4 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/requires.txt +23 -0
- evigauge_trace-0.1.0a25/evigauge_trace.egg-info/top_level.txt +4 -0
- evigauge_trace-0.1.0a25/pxcore/__init__.py +132 -0
- evigauge_trace-0.1.0a25/pxcore/agent_tools.py +50 -0
- evigauge_trace-0.1.0a25/pxcore/calibration/__init__.py +105 -0
- evigauge_trace-0.1.0a25/pxcore/calibration/battery.py +168 -0
- evigauge_trace-0.1.0a25/pxcore/calibration/profiles/claude-fable-5.json +22 -0
- evigauge_trace-0.1.0a25/pxcore/classifier.py +122 -0
- evigauge_trace-0.1.0a25/pxcore/drift.py +59 -0
- evigauge_trace-0.1.0a25/pxcore/fidelity.py +35 -0
- evigauge_trace-0.1.0a25/pxcore/gate.py +46 -0
- evigauge_trace-0.1.0a25/pxcore/integrations.py +172 -0
- evigauge_trace-0.1.0a25/pxcore/meter.py +39 -0
- evigauge_trace-0.1.0a25/pxcore/ops.py +59 -0
- evigauge_trace-0.1.0a25/pxcore/renderer/__init__.py +157 -0
- evigauge_trace-0.1.0a25/pxcore/renderer/font.py +131 -0
- evigauge_trace-0.1.0a25/pxcore/renderer/png.py +49 -0
- evigauge_trace-0.1.0a25/pxcore/types.py +129 -0
- evigauge_trace-0.1.0a25/pxcore_mcp/__init__.py +15 -0
- evigauge_trace-0.1.0a25/pxcore_mcp/__main__.py +34 -0
- evigauge_trace-0.1.0a25/pxcore_mcp/server.py +66 -0
- evigauge_trace-0.1.0a25/pxcore_mcp/tools.py +120 -0
- evigauge_trace-0.1.0a25/pxcore_mcp/transport.py +71 -0
- evigauge_trace-0.1.0a25/pxcore_proxy/__init__.py +12 -0
- evigauge_trace-0.1.0a25/pxcore_proxy/__main__.py +24 -0
- evigauge_trace-0.1.0a25/pxcore_proxy/proxy.py +115 -0
- evigauge_trace-0.1.0a25/pxcore_proxy/rewrite.py +16 -0
- evigauge_trace-0.1.0a25/pyproject.toml +253 -0
- evigauge_trace-0.1.0a25/setup.cfg +4 -0
- evigauge_trace-0.1.0a25/tests/test_wire_error_capture.py +66 -0
|
@@ -0,0 +1,1230 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evigauge-trace
|
|
3
|
+
Version: 0.1.0a25
|
|
4
|
+
Summary: Pluggable reasoning-trace observability for LLM applications
|
|
5
|
+
Author: Evigauge
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/evigauge-org/evigauge-python-sdk
|
|
8
|
+
Project-URL: Repository, https://github.com/evigauge-org/evigauge-python-sdk
|
|
9
|
+
Project-URL: Issues, https://github.com/evigauge-org/evigauge-python-sdk/issues
|
|
10
|
+
Keywords: observability,opentelemetry,llm,reasoning,tracing,otel
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: System :: Monitoring
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
Requires-Dist: opentelemetry-api<2.0,>=1.24
|
|
24
|
+
Requires-Dist: opentelemetry-sdk<2.0,>=1.24
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp<2.0,>=1.24
|
|
26
|
+
Requires-Dist: pydantic<3.0,>=2.6
|
|
27
|
+
Requires-Dist: tiktoken<1.0,>=0.6
|
|
28
|
+
Requires-Dist: httpx<1.0,>=0.27
|
|
29
|
+
Provides-Extra: shipcheck
|
|
30
|
+
Requires-Dist: pyyaml<7,>=6; extra == "shipcheck"
|
|
31
|
+
Provides-Extra: live
|
|
32
|
+
Requires-Dist: rich<15,>=13; extra == "live"
|
|
33
|
+
Provides-Extra: openllmetry
|
|
34
|
+
Requires-Dist: traceloop-sdk<1.0,>=0.18; extra == "openllmetry"
|
|
35
|
+
Provides-Extra: litellm
|
|
36
|
+
Requires-Dist: litellm<2.0,>=1.40; extra == "litellm"
|
|
37
|
+
Provides-Extra: adapters-microsoft
|
|
38
|
+
Provides-Extra: adapters-langchain
|
|
39
|
+
Requires-Dist: langchain-core<1.0,>=0.2; extra == "adapters-langchain"
|
|
40
|
+
|
|
41
|
+
# evigauge-trace
|
|
42
|
+
|
|
43
|
+
Pluggable reasoning-trace observability for LLM applications.
|
|
44
|
+
|
|
45
|
+
`evigauge-trace` is the **Python** client SDK for the **Evigauge Observability
|
|
46
|
+
Layer**. It captures *reasoning-altitude* spans — the decisions an agent made,
|
|
47
|
+
the sources it used, what each step cost, and the inputs to a reliability
|
|
48
|
+
score — and ships them to an Evigauge backend over OTLP. Unlike raw LLM logging,
|
|
49
|
+
the unit of observation is the *reasoning step*, not the HTTP call.
|
|
50
|
+
|
|
51
|
+
> **The package now includes more than tracing.** One `pip install evigauge-trace`
|
|
52
|
+
> also gives you three tools that ship with the SDK — all covered below:
|
|
53
|
+
> - **`evigauge shipcheck`** — a zero-LLM pre-merge CI gate that catches prompt /
|
|
54
|
+
> model / cost / cache regressions from the traces you already sent.
|
|
55
|
+
> - **`evigauge audit`** — a local, zero-egress security audit + map of an agentic
|
|
56
|
+
> app (MCP servers, agents, tools, hooks), graded against the NSA's MCP
|
|
57
|
+
> security guidance.
|
|
58
|
+
> - **`import pxcore`** (+ `pxcore-mcp` / `pxcore-proxy`) — deterministic
|
|
59
|
+
> image-as-context **token compression**: render dense reference context as
|
|
60
|
+
> images the model reads with native vision, keep exact content as text.
|
|
61
|
+
>
|
|
62
|
+
> See [What's new (0.1.0a13 → a16)](#whats-new-in-010a13--a16) and the
|
|
63
|
+
> [Also in this package](#also-in-this-package--shipcheck-audit-pxcore) section.
|
|
64
|
+
|
|
65
|
+
> **Not a Python shop?** Evigauge's ingest speaks plain OTLP/HTTP, so any
|
|
66
|
+
> language with an OpenTelemetry SDK works today — including Next.js /
|
|
67
|
+
> TypeScript and Vercel AI SDK apps. See [Use from TypeScript /
|
|
68
|
+
> Next.js](#use-from-typescript--nextjs) below. You don't need to spin up
|
|
69
|
+
> Python just to ship spans to us.
|
|
70
|
+
|
|
71
|
+
## What Evigauge does with your spans (the intelligence layer)
|
|
72
|
+
|
|
73
|
+
`evigauge-trace` is **not a logging service**. It is the client half of an
|
|
74
|
+
*AI-pipeline reconstruction engine*. Once your spans land in our ingest,
|
|
75
|
+
reconstruction engines run server-side on every batch:
|
|
76
|
+
|
|
77
|
+
| Engine | Reads from your spans | Produces |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| **Decomposition** | parent-child tree + `reasoning_role` | how the query was broken down + decomposition quality score |
|
|
80
|
+
| **Decision Trace** | `evigauge.decision.*` attributes per step | reasoning DAG — which branches were taken vs abandoned, and why |
|
|
81
|
+
| **Sources** | `evigauge.sources.used / consulted / dropped` URL lists | sources matrix + **live Exa-backed web verification** + per-domain authority prior that learns over time per workspace |
|
|
82
|
+
| **Reliability v2** | LLM completions + cited sources | per-claim hallucination score |
|
|
83
|
+
| **Correlation** | model, tokens, cost, pipeline shape | over-provisioning signals — where a premium model ran a simple query, and the measured saving of a cheaper one |
|
|
84
|
+
| **Savings Advisor** | prompt/context token structure | prompt-caching and context-optimization recommendations (advice only; never actuates) |
|
|
85
|
+
| **Alignment** | completions + your workspace knowledge base | whether answers align with your own source-of-truth documents |
|
|
86
|
+
| **External Cross-Check** | extracted claims + live web | independent verification of stage-1 claims |
|
|
87
|
+
|
|
88
|
+
Some engines are per-workspace toggles and some ship dark; ask your workspace owner
|
|
89
|
+
what is enabled.
|
|
90
|
+
|
|
91
|
+
You do **not** call any "compute" endpoint. Spans go in → artifacts appear in
|
|
92
|
+
the Read API seconds later. The semantic conventions on each span are what
|
|
93
|
+
unlock this — a raw "LLM call, 1.2s, 800 tokens" span gives you cost and
|
|
94
|
+
latency only; the rest of the magic needs the attributes documented in
|
|
95
|
+
[Semantic conventions](#semantic-conventions-for-non-python-languages) below.
|
|
96
|
+
|
|
97
|
+
## Install
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install evigauge-trace
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Already have it? **Upgrade to `0.1.0a16` or later** — you get Ship Check, the
|
|
104
|
+
agent audit, and pxcore, and every release before `0.1.0a12` silently discards
|
|
105
|
+
its auto-instrumented spans (see [What's new](#whats-new-in-010a13--a16)):
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
pip install --upgrade --pre evigauge-trace
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The `--pre` is required. Every published version is a pre-release
|
|
112
|
+
(`0.1.0a1` … `0.1.0a12`) and `pip` will not upgrade between pre-releases without
|
|
113
|
+
it. To see what you actually have versus what exists:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip show evigauge-trace # installed
|
|
117
|
+
pip index versions --pre evigauge-trace # available
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## What's new in `0.1.0a13` → `a16`
|
|
121
|
+
|
|
122
|
+
The SDK grew three capabilities beyond tracing, all shipped in this one package:
|
|
123
|
+
|
|
124
|
+
- **`0.1.0a13` — Ship Check.** `evigauge shipcheck`: a pre-merge CI gate that makes
|
|
125
|
+
**no LLM call and never re-runs your agent**. It projects a prompt/model/config
|
|
126
|
+
change against the traces you already sent — cost per call / trace / month,
|
|
127
|
+
prompt-cache safety, structure regressions — and a second 100%-local gate flags
|
|
128
|
+
Claude/agent artifacts (`CLAUDE.md`, `.mcp.json`, …) shipping inside your
|
|
129
|
+
production build. Also: prompt versioning + a Prompt Health Score, and the
|
|
130
|
+
`evigauge.prompt_id` / `evigauge.prompt_version` envelope keys for exact baseline
|
|
131
|
+
matching. See [Also in this package](#also-in-this-package--shipcheck-audit-pxcore).
|
|
132
|
+
- **`0.1.0a14` — Agent Security Audit.** `evigauge audit`: **local and zero-egress**
|
|
133
|
+
(no network, no LLM, nothing it finds leaves the machine). It maps an agentic
|
|
134
|
+
app's topology — agents, subagents, MCP servers, tools, hooks — plus a granular
|
|
135
|
+
reachability layer (capabilities, datastores, endpoints, secrets-by-name), and
|
|
136
|
+
audits it against the **NSA MCP Security CSI**: tool-description injection
|
|
137
|
+
(incl. hidden unicode), blanket OAuth scopes, unpinned `npx -y` boot-time code
|
|
138
|
+
execution, tool-name collisions, cleartext credentials, and secret→internet
|
|
139
|
+
exfiltration paths. Its headline is a committed `.evigauge/agentmap.lock` that
|
|
140
|
+
turns a silent capability change ("rug pull") into a reviewable PR diff.
|
|
141
|
+
- **`0.1.0a15`–`a16` — pxcore token compression.** `import pxcore`, plus the
|
|
142
|
+
`pxcore-mcp` and `pxcore-proxy` commands. Deterministic image-as-context
|
|
143
|
+
compression, folded into this distribution (no separate install). See
|
|
144
|
+
[pxcore](#pxcore--token-compression-import-pxcore) below.
|
|
145
|
+
- **`0.1.0a21` — Prompt-injection susceptibility audit.** A new check family in
|
|
146
|
+
`evigauge audit` (and Gate 3 of `evigauge shipcheck`). It reads **every system
|
|
147
|
+
instruction** in the repo — `CLAUDE.md`/`AGENTS.md`/`.cursorrules`, in-code
|
|
148
|
+
`system=`/`SYSTEM_PROMPT` literals, subagents, skills, `prompts:` files — and for
|
|
149
|
+
each gap names the injection **type** and the concrete **mitigation**, cited
|
|
150
|
+
per-check to OWASP LLM01:2025 / NIST AI 100-2e2025 / MITRE ATLAS / NSA-CISA.
|
|
151
|
+
Applicability-gated (a hardened prompt stays silent); a secret in a prompt is a
|
|
152
|
+
provable FAIL (value never emitted). Same **local, zero-egress, zero-LLM** contract;
|
|
153
|
+
the full report is local-only. See
|
|
154
|
+
[Also in this package](#also-in-this-package--shipcheck-audit-pxcore).
|
|
155
|
+
|
|
156
|
+
## Also in this package — shipcheck, audit, pxcore
|
|
157
|
+
|
|
158
|
+
These ship with `evigauge-trace`; nothing extra to install.
|
|
159
|
+
|
|
160
|
+
### Ship Check — `evigauge shipcheck` (zero-LLM PR gate)
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
pip install "evigauge-trace[shipcheck]" # the extra is just PyYAML, to read the policy file
|
|
164
|
+
evigauge shipcheck --help
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Declare a policy in `.evigauge/shipcheck.yml` (prompt files, cost thresholds), then
|
|
168
|
+
run `evigauge shipcheck` in CI. Exit `0` = pass, `1` = a gate failed the policy,
|
|
169
|
+
`2` = the check could not run (a backend outage is never a silent pass). It talks
|
|
170
|
+
to your workspace with `EVIGAUGE_API_KEY`; the artifact-leak gate is fully local and
|
|
171
|
+
needs no key (`evigauge shipcheck --skip-prompts`). Full guide: `docs/shipcheck.md`.
|
|
172
|
+
|
|
173
|
+
### Agent Security Audit — `evigauge audit` (local, zero-egress)
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
evigauge audit --map agentmap.html # writes a self-contained, offline HTML map
|
|
177
|
+
evigauge audit --approve-all # bless the current agent map as the baseline
|
|
178
|
+
evigauge audit --injection-report pi.md # prompt-injection susceptibility report (local only)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Runs automatically as a third gate of `evigauge shipcheck` too (WARN by default).
|
|
182
|
+
No API key, no network. Full spec: `docs/agent-map-security-audit.md`.
|
|
183
|
+
|
|
184
|
+
**Prompt-injection susceptibility (a21).** The same audit also checks whether each
|
|
185
|
+
**system instruction** is defenseless against injection arriving at runtime — and
|
|
186
|
+
names the injection **type** and the exact **mitigation** for every gap. It reads
|
|
187
|
+
`CLAUDE.md`/`AGENTS.md`/`.cursorrules`, in-code `system=`/`SYSTEM_PROMPT` literals,
|
|
188
|
+
subagents, skills, and your `prompts:` files (all read locally, sent nowhere).
|
|
189
|
+
Checks are **applicability-gated** so a hardened prompt stays silent; a secret in a
|
|
190
|
+
prompt is a provable FAIL (shape + location, never the value). Each finding cites
|
|
191
|
+
its own source (OWASP LLM01:2025, NIST AI 100-2e2025, MITRE ATLAS AML.T0051,
|
|
192
|
+
NSA/CISA). Configure it under `audit.injection` in `.evigauge/shipcheck.yml`:
|
|
193
|
+
|
|
194
|
+
```yaml
|
|
195
|
+
audit:
|
|
196
|
+
injection:
|
|
197
|
+
mode: warn # warn (default) | fail (exit 1, blocks the push) | off
|
|
198
|
+
report: prompt-injection-audit.md # LOCAL path for the full report
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The full report is an attacker's roadmap, so it is written **locally only**; the
|
|
202
|
+
shared PR comment gets the verdict + finding categories, never the evidence.
|
|
203
|
+
|
|
204
|
+
### pxcore — token compression (`import pxcore`)
|
|
205
|
+
|
|
206
|
+
Cut input tokens by rendering **dense, reference** context as images the model
|
|
207
|
+
reads with its native vision, while keeping exact content (ids, paths, hashes,
|
|
208
|
+
code you edit) as text. **No LLM in the compression path; nothing leaves the
|
|
209
|
+
machine.** It is **default-off per model** and images only where a calibration
|
|
210
|
+
battery has proven that model reads imaged text back reliably — everything else
|
|
211
|
+
stays text (safe).
|
|
212
|
+
|
|
213
|
+
**Python — library (compress context you assembled, or give your agent tools):**
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
import pxcore
|
|
217
|
+
from pxcore.calibration import load_profile
|
|
218
|
+
|
|
219
|
+
profile = load_profile("claude-fable-5")
|
|
220
|
+
blocks = pxcore.to_anthropic(big_reference_text, profile) # image+factsheet, or text
|
|
221
|
+
new_body, saved = pxcore.compress_anthropic(anthropic_request_body) # whole-request
|
|
222
|
+
|
|
223
|
+
from pxcore import agent_tools # ready-to-register agent tools
|
|
224
|
+
result = agent_tools.read("large/log.txt") # returns Anthropic content blocks
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
**TypeScript / Next.js — via the MCP server (no npm package needed):**
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
pxcore-mcp --http --port 8765 # your TS code connects with @modelcontextprotocol/sdk
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
**Claude Code — the in-path proxy (also images system prompt + history):**
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
pxcore-proxy # then launch with ANTHROPIC_BASE_URL=http://127.0.0.1:8788
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
Two one-time gates before it actually compresses (until then it safely returns
|
|
240
|
+
text): **calibrate your model**, and **confirm image passthrough**. Full guide:
|
|
241
|
+
`docs/frontend-pxcore-guide.md` and `docs/pxcore-readme.md`.
|
|
242
|
+
|
|
243
|
+
## What's new in `0.1.0a12`
|
|
244
|
+
|
|
245
|
+
**Bug fix — auto-instrumented spans were being discarded.** `init(auto_instrument=True)`
|
|
246
|
+
produced spans that the ingest API rejected and dead-lettered, silently, behind an
|
|
247
|
+
HTTP `202`. Two independent causes, both fixed:
|
|
248
|
+
|
|
249
|
+
1. **The `evigauge.*` envelope was never stamped.** `@observe` and `ReasoningTrace`
|
|
250
|
+
attached `evigauge.schema_version` / `org_id` / `workspace_id` / `project_id` to
|
|
251
|
+
each span; the auto-instrument wrappers did not. Ingest validates the span's own
|
|
252
|
+
attributes, so every one of them failed as `schema_version_unknown`.
|
|
253
|
+
2. **Three attributes the envelope schema forbids.** `EvigaugeSpanAttributes` is
|
|
254
|
+
`extra="forbid"`, so a single unknown `evigauge.*` key rejects the *entire* span.
|
|
255
|
+
`evigauge.cost.unpriced_model`, `evigauge.instrumentation_error` (auto-instrument) and
|
|
256
|
+
`evigauge.domain` (`ReasoningTrace(domain=…)`) are now emitted under the
|
|
257
|
+
`evigauge_sdk.` prefix, which ingest ignores rather than validates.
|
|
258
|
+
|
|
259
|
+
> `ReasoningTrace(domain=…)` no longer destroys the span. It is still **not** a
|
|
260
|
+
> queryable dimension server-side — Evigauge does not index `domain`.
|
|
261
|
+
|
|
262
|
+
**New — `transport="direct"`.** Export spans straight to `backend_url` over HTTPS. No
|
|
263
|
+
OpenTelemetry collector, no gRPC:
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
init(..., transport="direct") # or set EVIGAUGE_TRANSPORT=direct
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The **default is unchanged** (`transport="collector"`, OTLP/gRPC to
|
|
270
|
+
`collector_endpoint`), so upgrading does not alter the behaviour of an existing
|
|
271
|
+
deployment. If you keep the collector, set `encoding: json` on its `otlphttp`
|
|
272
|
+
exporter — Evigauge's ingest answers OTLP/**protobuf** with a `415`.
|
|
273
|
+
|
|
274
|
+
**Fixed — `init()` raised on Windows.** The WAL was opened for append before the
|
|
275
|
+
previous run's WAL was replayed, and renaming an open file raises `PermissionError`
|
|
276
|
+
on Windows (it silently succeeds on POSIX). Replay now happens first.
|
|
277
|
+
|
|
278
|
+
**Fixed — a cold-connection timeout dropped a span batch.** `BatchSpanProcessor` does
|
|
279
|
+
not retry a failed export. The direct exporter now retries once on transport errors.
|
|
280
|
+
|
|
281
|
+
## Quickstart
|
|
282
|
+
|
|
283
|
+
Call `init()` once at process startup:
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
from evigauge.trace import init
|
|
287
|
+
|
|
288
|
+
init(
|
|
289
|
+
org_id="pwc",
|
|
290
|
+
workspace_id="chatpwc",
|
|
291
|
+
project_id="due-diligence",
|
|
292
|
+
backend_url="https://ingest.evigauge.ai",
|
|
293
|
+
api_key="opx_live_...", # your workspace API key
|
|
294
|
+
transport="direct", # no collector to run
|
|
295
|
+
)
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
With `auto_instrument=True` (the default), every LLM call made through
|
|
299
|
+
litellm / anthropic / openai is now traced. No other code changes required.
|
|
300
|
+
|
|
301
|
+
## The three integration patterns
|
|
302
|
+
|
|
303
|
+
### Pattern A — Auto-instrument (zero code changes)
|
|
304
|
+
|
|
305
|
+
`init(auto_instrument=True)` monkey-patches the litellm / anthropic / openai
|
|
306
|
+
client methods at startup. Every completion call emits a `gen_ai.*` span
|
|
307
|
+
carrying the model, token usage, and computed USD cost. This is the default;
|
|
308
|
+
you get it just by calling `init()`.
|
|
309
|
+
|
|
310
|
+
### Pattern B — the `@observe` decorator
|
|
311
|
+
|
|
312
|
+
Wrap any function — sync or async — to emit a reasoning span for it:
|
|
313
|
+
|
|
314
|
+
```python
|
|
315
|
+
from evigauge.trace import observe
|
|
316
|
+
|
|
317
|
+
@observe(reasoning_role="decomposer", node_type="decomposer", name="plan.decompose")
|
|
318
|
+
async def decompose(query: str) -> list[str]:
|
|
319
|
+
...
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Nested `@observe` calls auto-parent via OpenTelemetry context — a decorated
|
|
323
|
+
function called inside another decorated function becomes its child span, so
|
|
324
|
+
the reasoning tree falls out of normal call structure.
|
|
325
|
+
|
|
326
|
+
### Pattern C — the `ReasoningTrace` context manager
|
|
327
|
+
|
|
328
|
+
For explicit, structured traces — when you want to record decisions, sources,
|
|
329
|
+
and costs by hand:
|
|
330
|
+
|
|
331
|
+
```python
|
|
332
|
+
from evigauge.trace import ReasoningTrace
|
|
333
|
+
|
|
334
|
+
with ReasoningTrace(name="answer.query", reasoning_role="synthesis") as trace:
|
|
335
|
+
trace.record_cost(model="claude-opus-4-6", input_tokens=1200, output_tokens=400)
|
|
336
|
+
trace.record_sources(used=[...], consulted=[...], dropped=[...])
|
|
337
|
+
trace.record_decision(selected="opt_a", rules_fired=[...], scores={...})
|
|
338
|
+
with trace.subnode(name="retrieve", node_type="retriever") as node:
|
|
339
|
+
node.record_sources(...)
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
### Framework adapters
|
|
343
|
+
|
|
344
|
+
For agent frameworks, one line instruments a whole crew:
|
|
345
|
+
|
|
346
|
+
```python
|
|
347
|
+
from evigauge.trace.adapters.microsoft import instrument_microsoft_agents
|
|
348
|
+
instrument_microsoft_agents(crew) # Microsoft Agent Framework
|
|
349
|
+
|
|
350
|
+
from evigauge.trace.adapters.reasonix import instrument_reasonix
|
|
351
|
+
instrument_reasonix(orchestrator) # Reasonix orchestrator
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
**CrewAI** — one call instruments the whole crew (root + one span per agent, with
|
|
355
|
+
tool calls nested under the agent that made them):
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
import os
|
|
359
|
+
os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" # BEFORE importing crewai (see note)
|
|
360
|
+
from evigauge.trace import init
|
|
361
|
+
init(org_id=..., workspace_id=..., project_id=..., backend_url=..., api_key=...,
|
|
362
|
+
transport="direct", auto_instrument=False) # leave auto_instrument OFF for CrewAI
|
|
363
|
+
|
|
364
|
+
from crewai import Crew
|
|
365
|
+
from evigauge.trace.adapters import instrument_crewai
|
|
366
|
+
crew = Crew(agents=[...], tasks=[...])
|
|
367
|
+
instrument_crewai(crew) # then kick off as usual
|
|
368
|
+
crew.kickoff(inputs={...})
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
> **You must set `CREWAI_DISABLE_TELEMETRY=true` before importing crewai.** CrewAI
|
|
372
|
+
> registers its own global OpenTelemetry provider; OTel allows exactly one, so if
|
|
373
|
+
> CrewAI's wins, evigauge's exporter is never attached and spans are dropped. The
|
|
374
|
+
> adapter warns you loudly if it detects this, but it can't undo it. Also keep
|
|
375
|
+
> `auto_instrument=False` — CrewAI's per-call LLM spans otherwise mis-name the
|
|
376
|
+
> fleet; the adapter puts total token usage on the root from `crew.usage_metrics`.
|
|
377
|
+
|
|
378
|
+
**LangChain** — a callback handler; pass it per run or register it globally:
|
|
379
|
+
|
|
380
|
+
```python
|
|
381
|
+
from evigauge.trace.adapters import get_langchain_handler, instrument_langchain
|
|
382
|
+
|
|
383
|
+
handler = get_langchain_handler()
|
|
384
|
+
chain.invoke(inputs, config={"callbacks": [handler]}) # per-run
|
|
385
|
+
|
|
386
|
+
instrument_langchain() # or: best-effort global registration for every run
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
Token usage comes from LangChain's `on_llm_end`, so leave `auto_instrument=False`
|
|
390
|
+
here too. Retriever calls emit a `retrieval` node with `evigauge.sources` populated.
|
|
391
|
+
|
|
392
|
+
Adapters are duck-typed — the CrewAI one imports nothing from crewai, and the
|
|
393
|
+
LangChain one imports its base class lazily, so `import evigauge.trace.adapters`
|
|
394
|
+
never fails for a project that doesn't use them, and no version is pinned.
|
|
395
|
+
|
|
396
|
+
## Per-employee attribution — `evigauge.end_user`
|
|
397
|
+
|
|
398
|
+
When your app serves many employees, tag each trace with the actor who made
|
|
399
|
+
the request. The alignment engine groups its on-/off-mission verdicts by this
|
|
400
|
+
key (and buckets unattributed traces under `(unattributed)`). It's optional and
|
|
401
|
+
per-request — set it wherever you already set `reasoning_role`:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
# On the decorator:
|
|
405
|
+
@observe(reasoning_role="synthesis", end_user="alice@acme.com")
|
|
406
|
+
def handle_query(...): ...
|
|
407
|
+
|
|
408
|
+
# On a ReasoningTrace / subnode:
|
|
409
|
+
with ReasoningTrace(name="chat.turn", end_user="alice@acme.com") as trace:
|
|
410
|
+
...
|
|
411
|
+
|
|
412
|
+
# Or imperatively, once the actor is resolved mid-request:
|
|
413
|
+
with ReasoningTrace(name="chat.turn") as trace:
|
|
414
|
+
trace.record_end_user(current_user.email) # parity with TS setEvigaugeEndUser
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
Distinct from the generic `evigauge.user_id` envelope tag — `end_user` is the
|
|
418
|
+
dedicated actor key the alignment feature reads. An empty/omitted value is a
|
|
419
|
+
no-op (the trace stays unattributed). (TypeScript: use `setEvigaugeEndUser(span,
|
|
420
|
+
endUser)` — see the instrumentation snippet below.)
|
|
421
|
+
|
|
422
|
+
## Agent Fleet Monitor — real-time topology & flags
|
|
423
|
+
|
|
424
|
+
The **Agent Fleet Monitor** is a *separate, real-time* capability from the batch
|
|
425
|
+
reconstruction engines above. It folds your live span stream into **one
|
|
426
|
+
persistent topology map per fleet** — nodes are your agents, edges are the
|
|
427
|
+
hand-offs between them — and shows **data-volume in/out** (tokens + cache
|
|
428
|
+
tokens) per node and edge, flagging **latency spikes** and **errors** live on
|
|
429
|
+
the dashboard. It is deterministic and **LLM-free**: it never reads your
|
|
430
|
+
prompt/response *content*, only counts and status.
|
|
431
|
+
|
|
432
|
+
**It works with zero changes** — it auto-discovers agents from the spans you
|
|
433
|
+
already emit. But because it identifies a node by this ladder…
|
|
434
|
+
|
|
435
|
+
```
|
|
436
|
+
gen_ai.agent.name → gen_ai.system | evigauge.reasoning_role | evigauge.node_type → (unattributed)
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
…if you don't name your agents, distinct agents that share a role collapse into
|
|
440
|
+
one coarse node (e.g. a fleet shows up as `retrieval|retrieval` instead of
|
|
441
|
+
`Planner`, `Researcher`, `Writer`). **To get a clean, correct topology, set a
|
|
442
|
+
stable `gen_ai.agent.name` on each agent's span.** That single attribute is the
|
|
443
|
+
whole ask.
|
|
444
|
+
|
|
445
|
+
### What the monitor reads (set these; most are already emitted)
|
|
446
|
+
|
|
447
|
+
| Signal | Attribute(s) | Who sets it | Effect |
|
|
448
|
+
|---|---|---|---|
|
|
449
|
+
| **Node identity** | `gen_ai.agent.name` (fallbacks: `agent.name`, `gen_ai.agent.id`, `agent.id`) | **you** (explicit) | The node label. Keep it **stable** across runs. |
|
|
450
|
+
| **Edge (A → B hand-off)** | `parent_span_id` | OTel automatically | Every hand-off = one edge. Nest your spans (see below). |
|
|
451
|
+
| **Volume in/out** | `gen_ai.usage.input_tokens` / `output_tokens` | auto-instrument / your LLM span | tokens-in / tokens-out on the node + edge |
|
|
452
|
+
| **Cache volume** | `gen_ai.usage.cache_read_input_tokens` / `cache_creation_input_tokens` (OpenAI `cached_tokens` also read) | auto-instrument | cache-in / cache-out |
|
|
453
|
+
| **Error flag** | span **status = ERROR** | your `try/except` → record on span | node/edge lights red |
|
|
454
|
+
| **Latency flag** | span duration | OTel automatically | amber flag when a node runs slow vs its own learned baseline |
|
|
455
|
+
|
|
456
|
+
> Fleet grouping in v1 is automatic: each run is grouped by its **entry
|
|
457
|
+
> (first-seen) agent**, so naming the *first* agent in a run especially matters.
|
|
458
|
+
|
|
459
|
+
### The one rule that determines the map: **name the span that carries the tokens**
|
|
460
|
+
|
|
461
|
+
The monitor folds **every span** into a node keyed by its identity, and a node's
|
|
462
|
+
**token/cache volume is the usage on the spans that share that identity.** The
|
|
463
|
+
in/out token counts live on your **LLM-call spans** (auto-instrument /
|
|
464
|
+
`traceLlmCall` / Vercel AI SDK) — *not* on a wrapper span. So:
|
|
465
|
+
|
|
466
|
+
- Set `gen_ai.agent.name` **on the LLM-call span itself** → that call's volume
|
|
467
|
+
folds onto the agent's node. This is what you want.
|
|
468
|
+
- Leave LLM-call spans unnamed → they group into a node named after the
|
|
469
|
+
provider (`anthropic`, `openai`) via the `gen_ai.system` fallback, and the
|
|
470
|
+
volume lands *there*. Still a valid view — just per-provider, not per-agent.
|
|
471
|
+
- Edges come from `parent_span_id`: nest spans so agent B's work runs inside
|
|
472
|
+
agent A's span, and you get the A → B hand-off edge for free.
|
|
473
|
+
|
|
474
|
+
### Python
|
|
475
|
+
|
|
476
|
+
**Recommended — keep auto-instrument on (zero per-call code).** Auto-instrument
|
|
477
|
+
already emits one span per LLM call carrying the token/cache volume. Name your
|
|
478
|
+
*agents* with `@observe` (or `ReasoningTrace`) and you get the hand-off topology
|
|
479
|
+
for free — nested calls auto-parent into edges:
|
|
480
|
+
|
|
481
|
+
```python
|
|
482
|
+
from evigauge.trace import init, observe
|
|
483
|
+
from opentelemetry.trace import get_current_span
|
|
484
|
+
|
|
485
|
+
init(org_id="acme", workspace_id="chat", project_id="ui",
|
|
486
|
+
backend_url="https://evigauge.internal.example.com", api_key="opx_live_…")
|
|
487
|
+
# auto_instrument=True is the default -> LLM calls emit their own spans.
|
|
488
|
+
|
|
489
|
+
@observe(reasoning_role="planner", node_type="planner", name="plan.step")
|
|
490
|
+
async def planner(question):
|
|
491
|
+
get_current_span().set_attribute("gen_ai.agent.name", "Planner") # STABLE label
|
|
492
|
+
plan = await ask_llm(f"Plan: {question}")
|
|
493
|
+
return await researcher(plan) # child span -> Planner → Researcher edge
|
|
494
|
+
|
|
495
|
+
@observe(reasoning_role="retriever", node_type="researcher", name="research.step")
|
|
496
|
+
async def researcher(plan):
|
|
497
|
+
get_current_span().set_attribute("gen_ai.agent.name", "Researcher")
|
|
498
|
+
return await ask_llm(plan) # auto-instrumented LLM call nests here
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
**What you get:** agent nodes `Planner` and `Researcher` with the hand-off edge
|
|
502
|
+
between them, plus the LLM calls as **provider nodes** (`anthropic`, `openai`)
|
|
503
|
+
that carry the token/cache volume, hanging off whichever agent made them. This
|
|
504
|
+
is the low-friction path and it keeps cost + reliability + decomposition
|
|
505
|
+
instrumentation fully intact.
|
|
506
|
+
|
|
507
|
+
**Advanced — per-agent volume (fold the tokens onto the *agent* node).** The
|
|
508
|
+
token counts live on the LLM-call span, so to move volume off the provider node
|
|
509
|
+
and onto the agent node, `gen_ai.agent.name` has to be on *that* span. Since
|
|
510
|
+
auto-instrument owns it and doesn't set an agent name, you hand-instrument the
|
|
511
|
+
call — and turn auto-instrument off (`init(..., auto_instrument=False)`) **or**
|
|
512
|
+
skip it for these calls, so the same tokens aren't also counted on a duplicate
|
|
513
|
+
`anthropic` node:
|
|
514
|
+
|
|
515
|
+
```python
|
|
516
|
+
from opentelemetry.trace import get_tracer
|
|
517
|
+
tracer = get_tracer("my-ai-app")
|
|
518
|
+
|
|
519
|
+
def call_as_agent(agent_name, role, messages):
|
|
520
|
+
with tracer.start_as_current_span(f"agent.{agent_name}") as span:
|
|
521
|
+
span.set_attribute("gen_ai.agent.name", agent_name) # node identity
|
|
522
|
+
span.set_attribute("evigauge.reasoning_role", role)
|
|
523
|
+
r = anthropic_client.messages.create(
|
|
524
|
+
model="claude-opus-4-6", max_tokens=800, messages=messages)
|
|
525
|
+
span.set_attribute("gen_ai.usage.input_tokens", r.usage.input_tokens)
|
|
526
|
+
span.set_attribute("gen_ai.usage.output_tokens", r.usage.output_tokens)
|
|
527
|
+
return r
|
|
528
|
+
```
|
|
529
|
+
|
|
530
|
+
> Trade-off: `auto_instrument=False` is **global** — it disables the automatic
|
|
531
|
+
> cost/reliability spans everywhere, not just for the fleet map. Only reach for
|
|
532
|
+
> the advanced path if per-agent *volume* attribution matters more than
|
|
533
|
+
> zero-effort cost tracking. For most teams the recommended path (provider nodes
|
|
534
|
+
> carry volume, agent nodes carry structure) is the right call.
|
|
535
|
+
|
|
536
|
+
Errors flag automatically — a raised exception sets the span status to ERROR,
|
|
537
|
+
which the monitor turns into a red flag. No extra code.
|
|
538
|
+
|
|
539
|
+
### Next.js / TypeScript
|
|
540
|
+
|
|
541
|
+
Put `gen_ai.agent.name` on the **token-bearing** span. Extend the `traceLlmCall`
|
|
542
|
+
helper from [Pattern B](#pattern-b--raw-anthropic-aisdk--openai-calls-auto-instrument-equivalent)
|
|
543
|
+
with an `agentName` — one line — so identity and volume land on the same node:
|
|
544
|
+
|
|
545
|
+
```ts
|
|
546
|
+
// lib/evigauge.ts — add agentName to the existing helper
|
|
547
|
+
export async function traceLlmCall<T>(
|
|
548
|
+
fn: () => Promise<T>,
|
|
549
|
+
opts: { system: System; model: string; agentName?: string },
|
|
550
|
+
): Promise<T> {
|
|
551
|
+
return tracer.startActiveSpan(`gen_ai.${opts.system}.completion`, async (span) => {
|
|
552
|
+
if (opts.agentName) span.setAttribute("gen_ai.agent.name", opts.agentName); // node label + volume
|
|
553
|
+
span.setAttribute("gen_ai.system", opts.system);
|
|
554
|
+
span.setAttribute("gen_ai.request.model", opts.model);
|
|
555
|
+
// ...unchanged: attachUsage() stamps gen_ai.usage.* -> this node's volume,
|
|
556
|
+
// recordException + ERROR status -> red flag, span.end() in finally.
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
```
|
|
560
|
+
|
|
561
|
+
```ts
|
|
562
|
+
// app/api/answer/route.ts — two-agent fleet: Planner → Researcher
|
|
563
|
+
import { traceLlmCall } from "@/lib/evigauge";
|
|
564
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
565
|
+
|
|
566
|
+
const anthropic = new Anthropic();
|
|
567
|
+
|
|
568
|
+
export async function POST(req: Request) {
|
|
569
|
+
const { question } = await req.json();
|
|
570
|
+
|
|
571
|
+
// Planner's call is the parent span...
|
|
572
|
+
const plan = await traceLlmCall(
|
|
573
|
+
() => anthropic.messages.create({
|
|
574
|
+
model: "claude-opus-4-6", max_tokens: 200,
|
|
575
|
+
messages: [{ role: "user", content: `Plan: ${question}` }],
|
|
576
|
+
}),
|
|
577
|
+
{ system: "anthropic", model: "claude-opus-4-6", agentName: "Planner" },
|
|
578
|
+
);
|
|
579
|
+
|
|
580
|
+
// ...Researcher's call nests inside it -> Planner → Researcher edge.
|
|
581
|
+
const answer = await traceLlmCall(
|
|
582
|
+
() => anthropic.messages.create({
|
|
583
|
+
model: "claude-opus-4-6", max_tokens: 800,
|
|
584
|
+
messages: [{ role: "user", content: String(plan.content) }],
|
|
585
|
+
}),
|
|
586
|
+
{ system: "anthropic", model: "claude-opus-4-6", agentName: "Researcher" },
|
|
587
|
+
);
|
|
588
|
+
|
|
589
|
+
return Response.json({ answer: answer.content });
|
|
590
|
+
}
|
|
591
|
+
```
|
|
592
|
+
|
|
593
|
+
**Vercel AI SDK users:** each model call already emits its own span with the
|
|
594
|
+
token volume — wrap it in a `tracer.startActiveSpan(...)` on which you set
|
|
595
|
+
`gen_ai.agent.name`, so the AI SDK's usage folds onto the named agent node.
|
|
596
|
+
|
|
597
|
+
That's the whole integration: **name the token-bearing spans and keep the calls
|
|
598
|
+
nested.** Edges, volume, latency baselines, and error flags are derived for you.
|
|
599
|
+
Read it back over REST + WebSocket per
|
|
600
|
+
`docs/fleet-monitor-dashboard-integration.md`.
|
|
601
|
+
|
|
602
|
+
## Use from TypeScript / Next.js
|
|
603
|
+
|
|
604
|
+
`evigauge-trace` is the Python sugar. The transport is plain **OTLP/HTTP**, so
|
|
605
|
+
your Next.js / TypeScript / Vercel-AI-SDK / Cloudflare-Workers app ships to
|
|
606
|
+
the exact same ingest with the exact same backend intelligence — *no Python
|
|
607
|
+
process required*.
|
|
608
|
+
|
|
609
|
+
### Install (Next.js, Node runtime)
|
|
610
|
+
|
|
611
|
+
```bash
|
|
612
|
+
npm install \
|
|
613
|
+
@opentelemetry/api \
|
|
614
|
+
@opentelemetry/sdk-node \
|
|
615
|
+
@opentelemetry/exporter-trace-otlp-http \
|
|
616
|
+
@opentelemetry/resources \
|
|
617
|
+
@opentelemetry/semantic-conventions
|
|
618
|
+
```
|
|
619
|
+
|
|
620
|
+
### One-time setup — `instrumentation.ts` (Next.js native hook)
|
|
621
|
+
|
|
622
|
+
```ts
|
|
623
|
+
// instrumentation.ts (project root, sibling of next.config.js)
|
|
624
|
+
import { NodeSDK } from "@opentelemetry/sdk-node";
|
|
625
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
626
|
+
import { Resource } from "@opentelemetry/resources";
|
|
627
|
+
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
|
|
628
|
+
|
|
629
|
+
export async function register() {
|
|
630
|
+
if (process.env.NEXT_RUNTIME !== "nodejs") return;
|
|
631
|
+
|
|
632
|
+
const sdk = new NodeSDK({
|
|
633
|
+
resource: new Resource({
|
|
634
|
+
[SemanticResourceAttributes.SERVICE_NAME]: "my-ai-app",
|
|
635
|
+
"evigauge.ai": process.env.EVIGAUGE_ORG_ID!,
|
|
636
|
+
"evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID!,
|
|
637
|
+
"evigauge.project": process.env.EVIGAUGE_PROJECT_ID!,
|
|
638
|
+
}),
|
|
639
|
+
traceExporter: new OTLPTraceExporter({
|
|
640
|
+
url: "https://ingest.evigauge.ai/v1/traces",
|
|
641
|
+
headers: { "x-evigauge-api-key": process.env.EVIGAUGE_API_KEY! },
|
|
642
|
+
}),
|
|
643
|
+
});
|
|
644
|
+
sdk.start();
|
|
645
|
+
}
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
Add to `next.config.js`:
|
|
649
|
+
|
|
650
|
+
```js
|
|
651
|
+
experimental: { instrumentationHook: true }
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
That's it — your app is now wired to Evigauge.
|
|
655
|
+
|
|
656
|
+
### Pattern A — Vercel AI SDK (recommended)
|
|
657
|
+
|
|
658
|
+
If you use the `ai` SDK (`generateText`, `streamText`, `generateObject`),
|
|
659
|
+
just enable its native OTel telemetry. Every model call becomes a span
|
|
660
|
+
automatically, and our reconstruction engines pick up cost / latency /
|
|
661
|
+
model / token usage for free:
|
|
662
|
+
|
|
663
|
+
```ts
|
|
664
|
+
import { generateText } from "ai";
|
|
665
|
+
import { openai } from "@ai-sdk/openai";
|
|
666
|
+
|
|
667
|
+
const result = await generateText({
|
|
668
|
+
model: openai("gpt-4o"),
|
|
669
|
+
prompt: userQuery,
|
|
670
|
+
experimental_telemetry: {
|
|
671
|
+
isEnabled: true,
|
|
672
|
+
functionId: "answer-user",
|
|
673
|
+
metadata: {
|
|
674
|
+
"evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID!,
|
|
675
|
+
"evigauge.trace_kind": "answer",
|
|
676
|
+
},
|
|
677
|
+
},
|
|
678
|
+
});
|
|
679
|
+
```
|
|
680
|
+
|
|
681
|
+
### Pattern B — raw `@anthropic-ai/sdk` / `openai` calls (auto-instrument equivalent)
|
|
682
|
+
|
|
683
|
+
Vercel AI SDK is great, but plenty of TS apps call `@anthropic-ai/sdk` or
|
|
684
|
+
`openai` directly. The Python SDK auto-instruments these for you; in TS you
|
|
685
|
+
write a tiny helper *once*, then every LLM call lights up the same way. The
|
|
686
|
+
helper emits the **same `gen_ai.*` semantic conventions** the Python SDK
|
|
687
|
+
emits (mirrors `evigauge_trace/evigauge/trace/auto_instrument.py`), so the
|
|
688
|
+
backend treats both identically.
|
|
689
|
+
|
|
690
|
+
**One-time helper** — `lib/evigauge.ts`:
|
|
691
|
+
|
|
692
|
+
```ts
|
|
693
|
+
import { trace, SpanStatusCode, Span } from "@opentelemetry/api";
|
|
694
|
+
|
|
695
|
+
type System = "anthropic" | "openai" | "litellm";
|
|
696
|
+
|
|
697
|
+
const tracer = trace.getTracer("evigauge-trace-helpers");
|
|
698
|
+
|
|
699
|
+
export async function traceLlmCall<T>(
|
|
700
|
+
fn: () => Promise<T>,
|
|
701
|
+
opts: { system: System; model: string },
|
|
702
|
+
): Promise<T> {
|
|
703
|
+
return tracer.startActiveSpan(
|
|
704
|
+
`gen_ai.${opts.system}.completion`,
|
|
705
|
+
async (span: Span) => {
|
|
706
|
+
span.setAttribute("gen_ai.system", opts.system);
|
|
707
|
+
span.setAttribute("gen_ai.request.model", opts.model);
|
|
708
|
+
try {
|
|
709
|
+
const resp = await fn();
|
|
710
|
+
attachUsage(span, resp, opts.system, opts.model);
|
|
711
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
712
|
+
return resp;
|
|
713
|
+
} catch (err) {
|
|
714
|
+
span.recordException(err as Error);
|
|
715
|
+
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
716
|
+
throw err;
|
|
717
|
+
} finally {
|
|
718
|
+
span.end();
|
|
719
|
+
}
|
|
720
|
+
},
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
function attachUsage(span: Span, resp: any, system: System, requested: string) {
|
|
725
|
+
try {
|
|
726
|
+
const usage = resp?.usage ?? {};
|
|
727
|
+
const inputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0;
|
|
728
|
+
const outputTokens = usage.output_tokens ?? usage.completion_tokens ?? 0;
|
|
729
|
+
const modelUsed = resp?.model ?? requested;
|
|
730
|
+
const finish =
|
|
731
|
+
resp?.choices?.[0]?.finish_reason ?? // OpenAI / LiteLLM
|
|
732
|
+
resp?.stop_reason ?? // Anthropic
|
|
733
|
+
"";
|
|
734
|
+
span.setAttribute("gen_ai.usage.input_tokens", Number(inputTokens));
|
|
735
|
+
span.setAttribute("gen_ai.usage.output_tokens", Number(outputTokens));
|
|
736
|
+
if (modelUsed) span.setAttribute("gen_ai.response.model", String(modelUsed));
|
|
737
|
+
if (finish) span.setAttribute("gen_ai.response.finish_reason", String(finish));
|
|
738
|
+
} catch (err) {
|
|
739
|
+
span.setAttribute("evigauge.instrumentation_error", String(err));
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
```
|
|
743
|
+
|
|
744
|
+
**Usage with `@anthropic-ai/sdk`** — wrap the call, no other changes:
|
|
745
|
+
|
|
746
|
+
```ts
|
|
747
|
+
// app/api/answer/route.ts
|
|
748
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
749
|
+
import { traceLlmCall } from "@/lib/evigauge";
|
|
750
|
+
|
|
751
|
+
const anthropic = new Anthropic();
|
|
752
|
+
|
|
753
|
+
export async function POST(req: Request) {
|
|
754
|
+
const { question } = await req.json();
|
|
755
|
+
const resp = await traceLlmCall(
|
|
756
|
+
() => anthropic.messages.create({
|
|
757
|
+
model: "claude-opus-4-6",
|
|
758
|
+
max_tokens: 100,
|
|
759
|
+
messages: [{ role: "user", content: question }],
|
|
760
|
+
}),
|
|
761
|
+
{ system: "anthropic", model: "claude-opus-4-6" },
|
|
762
|
+
);
|
|
763
|
+
return Response.json({ answer: resp.content });
|
|
764
|
+
}
|
|
765
|
+
```
|
|
766
|
+
|
|
767
|
+
**Usage with `openai`** — identical helper, only `system` changes:
|
|
768
|
+
|
|
769
|
+
```ts
|
|
770
|
+
import OpenAI from "openai";
|
|
771
|
+
import { traceLlmCall } from "@/lib/evigauge";
|
|
772
|
+
|
|
773
|
+
const openai = new OpenAI();
|
|
774
|
+
|
|
775
|
+
const resp = await traceLlmCall(
|
|
776
|
+
() => openai.chat.completions.create({
|
|
777
|
+
model: "gpt-4o",
|
|
778
|
+
messages: [{ role: "user", content: question }],
|
|
779
|
+
}),
|
|
780
|
+
{ system: "openai", model: "gpt-4o" },
|
|
781
|
+
);
|
|
782
|
+
```
|
|
783
|
+
|
|
784
|
+
What lands in Evigauge — the same span shape the Python SDK's auto-instrument
|
|
785
|
+
produces, so cost/latency/token attribution and the reliability engine work
|
|
786
|
+
out of the box:
|
|
787
|
+
|
|
788
|
+
| Attribute | Value |
|
|
789
|
+
|---|---|
|
|
790
|
+
| `gen_ai.system` | `anthropic` / `openai` / `litellm` |
|
|
791
|
+
| `gen_ai.request.model` | `claude-opus-4-6` |
|
|
792
|
+
| `gen_ai.response.model` | `claude-opus-4-6` (server-confirmed) |
|
|
793
|
+
| `gen_ai.usage.input_tokens` | 12 |
|
|
794
|
+
| `gen_ai.usage.output_tokens` | 8 |
|
|
795
|
+
| `gen_ai.response.finish_reason` | `end_turn` (Anthropic) / `stop` (OpenAI) |
|
|
796
|
+
|
|
797
|
+
Backend cost reconstruction reads `gen_ai.response.model` + the token
|
|
798
|
+
counts — no extra work needed on the client.
|
|
799
|
+
|
|
800
|
+
### Pattern C — manual reasoning spans (decomposer, decision, sources)
|
|
801
|
+
|
|
802
|
+
To unlock the **Decomposition / Decision Trace / Sources** engines you have
|
|
803
|
+
to set our semantic-convention attributes (see the reference table at the
|
|
804
|
+
bottom). Example — a decomposer step that also records the URLs it consulted
|
|
805
|
+
and the decision it made:
|
|
806
|
+
|
|
807
|
+
```ts
|
|
808
|
+
import { trace, SpanStatusCode } from "@opentelemetry/api";
|
|
809
|
+
|
|
810
|
+
const tracer = trace.getTracer("my-ai-app");
|
|
811
|
+
|
|
812
|
+
await tracer.startActiveSpan("plan.decompose", async (span) => {
|
|
813
|
+
span.setAttribute("evigauge.reasoning_role", "decomposer");
|
|
814
|
+
span.setAttribute("evigauge.node_type", "decomposer");
|
|
815
|
+
|
|
816
|
+
try {
|
|
817
|
+
const subqueries = await decomposeQuery(userQuery);
|
|
818
|
+
const consulted = await retrieveCandidates(subqueries);
|
|
819
|
+
const cited = pickCitations(consulted);
|
|
820
|
+
|
|
821
|
+
// Sources engine consumes these JSON arrays of URLs:
|
|
822
|
+
span.setAttribute("evigauge.sources.used", JSON.stringify(cited));
|
|
823
|
+
span.setAttribute("evigauge.sources.consulted", JSON.stringify(consulted));
|
|
824
|
+
span.setAttribute("evigauge.sources.dropped", JSON.stringify([]));
|
|
825
|
+
|
|
826
|
+
// Decision Trace engine consumes these:
|
|
827
|
+
span.setAttribute("evigauge.decision.selected", "strategy_a");
|
|
828
|
+
span.setAttribute("evigauge.decision.rules_fired", JSON.stringify(["recency_boost"]));
|
|
829
|
+
span.setAttribute("evigauge.decision.scores", JSON.stringify({ a: 0.82, b: 0.41 }));
|
|
830
|
+
|
|
831
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
832
|
+
return subqueries;
|
|
833
|
+
} catch (err) {
|
|
834
|
+
span.recordException(err as Error);
|
|
835
|
+
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
836
|
+
throw err;
|
|
837
|
+
} finally {
|
|
838
|
+
span.end();
|
|
839
|
+
}
|
|
840
|
+
});
|
|
841
|
+
```
|
|
842
|
+
|
|
843
|
+
Nested `startActiveSpan` calls auto-parent the same way they do in Python —
|
|
844
|
+
the reasoning tree falls out of normal control flow.
|
|
845
|
+
|
|
846
|
+
### Edge runtime / Cloudflare Workers
|
|
847
|
+
|
|
848
|
+
Full OTel SDKs don't fit in Edge / Workers. Use the `fetch`-based exporter or
|
|
849
|
+
fall back to a `POST` to `/v1/traces` on a Node API route. (Edge-native SDK
|
|
850
|
+
is on the roadmap; ask if you need it sooner.)
|
|
851
|
+
|
|
852
|
+
## Multi-provider chat apps (UI dropdown picks the model)
|
|
853
|
+
|
|
854
|
+
Most production chat apps let the end-user pick a model from a dropdown —
|
|
855
|
+
Claude, GPT-4o, Gemini, etc. — at runtime. `evigauge-trace` handles this
|
|
856
|
+
without any extra config: **the patches are at the SDK-method level, not at
|
|
857
|
+
some global "current model" setting**. Whichever SDK your dispatcher
|
|
858
|
+
actually calls for a given turn is what gets tagged. Each turn → its own
|
|
859
|
+
span → correctly attributed by provider and model.
|
|
860
|
+
|
|
861
|
+
```
|
|
862
|
+
UI dropdown → your dispatcher → SDK A.method() ──┐
|
|
863
|
+
↘ SDK B.method() ──┼──► evigauge-trace tags
|
|
864
|
+
↘ SDK C.method() ──┘ each one individually
|
|
865
|
+
```
|
|
866
|
+
|
|
867
|
+
A single chat session with mixed turns shows up as N spans, one per call,
|
|
868
|
+
with `gen_ai.system` and `gen_ai.request.model` set per-turn. Cost rolls up
|
|
869
|
+
per provider, per model — no extra plumbing on your side.
|
|
870
|
+
|
|
871
|
+
### Python — explicit dispatcher
|
|
872
|
+
|
|
873
|
+
```python
|
|
874
|
+
from anthropic import Anthropic
|
|
875
|
+
from openai import OpenAI
|
|
876
|
+
from evigauge.trace import init
|
|
877
|
+
|
|
878
|
+
init(org_id="acme", workspace_id="chat", project_id="ui",
|
|
879
|
+
backend_url="…", api_key="opx_live_…")
|
|
880
|
+
|
|
881
|
+
anthropic_client = Anthropic()
|
|
882
|
+
openai_client = OpenAI()
|
|
883
|
+
|
|
884
|
+
def chat(user_selected_model: str, messages: list[dict]) -> str:
|
|
885
|
+
if user_selected_model.startswith("claude"):
|
|
886
|
+
r = anthropic_client.messages.create(
|
|
887
|
+
model=user_selected_model, max_tokens=1000, messages=messages)
|
|
888
|
+
return r.content[0].text
|
|
889
|
+
if user_selected_model.startswith("gpt"):
|
|
890
|
+
r = openai_client.chat.completions.create(
|
|
891
|
+
model=user_selected_model, messages=messages)
|
|
892
|
+
return r.choices[0].message.content
|
|
893
|
+
raise ValueError(f"unknown model: {user_selected_model}")
|
|
894
|
+
```
|
|
895
|
+
|
|
896
|
+
Each branch hits a patched method; spans land with the right `gen_ai.system`
|
|
897
|
+
(`anthropic` or `openai`) and `gen_ai.request.model`.
|
|
898
|
+
|
|
899
|
+
### Python — single-call dispatcher via `litellm` (recommended)
|
|
900
|
+
|
|
901
|
+
The cleanest pattern when you support many providers — one function, one
|
|
902
|
+
patched call, ~100 providers covered out of the box:
|
|
903
|
+
|
|
904
|
+
```python
|
|
905
|
+
import litellm
|
|
906
|
+
|
|
907
|
+
def chat(user_selected_model: str, messages: list[dict]) -> str:
|
|
908
|
+
r = litellm.completion(
|
|
909
|
+
model=user_selected_model, # e.g. "anthropic/claude-opus-4-6", "openai/gpt-4o", "gemini/gemini-pro"
|
|
910
|
+
messages=messages,
|
|
911
|
+
)
|
|
912
|
+
return r.choices[0].message.content
|
|
913
|
+
```
|
|
914
|
+
|
|
915
|
+
`evigauge-trace` patches `litellm.completion`, so every dropdown selection
|
|
916
|
+
routes through the same patched function. Spans get `gen_ai.system="litellm"`
|
|
917
|
+
and `gen_ai.request.model` carries the actual provider+model
|
|
918
|
+
(`anthropic/claude-opus-4-6`). Our cost engine handles the provider prefix.
|
|
919
|
+
|
|
920
|
+
### TypeScript — explicit dispatcher with `traceLlmCall`
|
|
921
|
+
|
|
922
|
+
The `traceLlmCall` helper from [Pattern B](#pattern-b--raw-anthropic-aisdk--openai-calls-auto-instrument-equivalent)
|
|
923
|
+
takes `{ system, model }` per call, so the dispatcher writes the right tag
|
|
924
|
+
per branch:
|
|
925
|
+
|
|
926
|
+
```ts
|
|
927
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
928
|
+
import OpenAI from "openai";
|
|
929
|
+
import { traceLlmCall } from "@/lib/evigauge";
|
|
930
|
+
|
|
931
|
+
const anthropic = new Anthropic();
|
|
932
|
+
const openai = new OpenAI();
|
|
933
|
+
|
|
934
|
+
export async function chat(selectedModel: string, messages: any[]) {
|
|
935
|
+
if (selectedModel.startsWith("claude")) {
|
|
936
|
+
return traceLlmCall(
|
|
937
|
+
() => anthropic.messages.create({ model: selectedModel, max_tokens: 1000, messages }),
|
|
938
|
+
{ system: "anthropic", model: selectedModel },
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
if (selectedModel.startsWith("gpt")) {
|
|
942
|
+
return traceLlmCall(
|
|
943
|
+
() => openai.chat.completions.create({ model: selectedModel, messages }),
|
|
944
|
+
{ system: "openai", model: selectedModel },
|
|
945
|
+
);
|
|
946
|
+
}
|
|
947
|
+
throw new Error(`unknown model: ${selectedModel}`);
|
|
948
|
+
}
|
|
949
|
+
```
|
|
950
|
+
|
|
951
|
+
### TypeScript — Vercel AI SDK provider lookup (recommended)
|
|
952
|
+
|
|
953
|
+
Each `@ai-sdk/*` provider package emits its own OTel span with `gen_ai.system`
|
|
954
|
+
set correctly, so a provider lookup table gives you correct attribution
|
|
955
|
+
across N providers with zero per-provider code:
|
|
956
|
+
|
|
957
|
+
```ts
|
|
958
|
+
import { generateText } from "ai";
|
|
959
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
960
|
+
import { openai } from "@ai-sdk/openai";
|
|
961
|
+
import { google } from "@ai-sdk/google";
|
|
962
|
+
|
|
963
|
+
const providers = { anthropic, openai, google } as const;
|
|
964
|
+
type Provider = keyof typeof providers;
|
|
965
|
+
|
|
966
|
+
export async function chat(provider: Provider, model: string, prompt: string) {
|
|
967
|
+
return generateText({
|
|
968
|
+
model: providers[provider](model),
|
|
969
|
+
prompt,
|
|
970
|
+
experimental_telemetry: {
|
|
971
|
+
isEnabled: true,
|
|
972
|
+
metadata: { "evigauge.workspace": process.env.EVIGAUGE_WORKSPACE_ID! },
|
|
973
|
+
},
|
|
974
|
+
});
|
|
975
|
+
}
|
|
976
|
+
```
|
|
977
|
+
|
|
978
|
+
### Bonus — capturing the dropdown choice itself
|
|
979
|
+
|
|
980
|
+
Useful for debugging "I picked Claude but the answer feels like GPT-4o" —
|
|
981
|
+
stamp the user's intent on a parent span so you can see both layers in the
|
|
982
|
+
trace tree:
|
|
983
|
+
|
|
984
|
+
```python
|
|
985
|
+
from evigauge.trace import ReasoningTrace
|
|
986
|
+
|
|
987
|
+
with ReasoningTrace(name="chat.turn", reasoning_role="synthesis") as trace:
|
|
988
|
+
trace._span.set_attribute("evigauge.user_selected_model", user_selected_model)
|
|
989
|
+
answer = chat(user_selected_model, messages)
|
|
990
|
+
```
|
|
991
|
+
|
|
992
|
+
The parent span carries what the user asked for; the child span carries
|
|
993
|
+
what was actually sent. If they diverge, your dispatcher has a bug.
|
|
994
|
+
|
|
995
|
+
## Cost for any model, any provider
|
|
996
|
+
|
|
997
|
+
`evigauge.cost.usd` is computed for **any model from any provider** — there is no
|
|
998
|
+
fixed allow-list. The SDK resolves a price through a layered strategy, first hit
|
|
999
|
+
wins:
|
|
1000
|
+
|
|
1001
|
+
1. **Provider-reported cost** — if the response already carries a cost (e.g.
|
|
1002
|
+
OpenRouter usage accounting, or litellm's `response_cost`), it is used
|
|
1003
|
+
as-is. Zero maintenance, exact for every model the gateway prices.
|
|
1004
|
+
2. **Bundled snapshot** — a version-pinned table of common models (GPT-4o,
|
|
1005
|
+
Claude, Kimi K2, Qwen2.5-72B, GLM-4.6, DeepSeek-V3, …), matched after
|
|
1006
|
+
normalizing the model id, so provider prefixes and variant suffixes resolve:
|
|
1007
|
+
`openrouter/z-ai/glm-4.6:free` → `glm-4.6`.
|
|
1008
|
+
3. **litellm price map** — if `litellm` is importable, its continuously-updated
|
|
1009
|
+
map (thousands of models across ~100 providers) is consulted. No network.
|
|
1010
|
+
4. **OpenRouter live** — last resort only: prices are fetched from OpenRouter's
|
|
1011
|
+
`/api/v1/models` and cached (6h). Disable with
|
|
1012
|
+
`EVIGAUGE_PRICING_DISABLE_NETWORK=1`.
|
|
1013
|
+
5. If none match, cost is `0.0` **and** `evigauge.cost.unpriced_model` is set — a
|
|
1014
|
+
missing price is always flagged, never a silent zero.
|
|
1015
|
+
|
|
1016
|
+
`evigauge.cost.model_pricing_version` records which source produced the figure
|
|
1017
|
+
(`provider`, the snapshot version, `litellm:<ver>`, or `openrouter-live`), so
|
|
1018
|
+
the cost is reproducible server-side. This works identically through OpenRouter
|
|
1019
|
+
or any gateway — pass the model as usual
|
|
1020
|
+
(`litellm.completion(model="openrouter/moonshotai/kimi-k2", …)`) and the span
|
|
1021
|
+
carries an accurate `evigauge.cost.usd`.
|
|
1022
|
+
|
|
1023
|
+
## Reading the computed artifacts — Read API
|
|
1024
|
+
|
|
1025
|
+
Once spans are in, your Next.js dashboard / debug UI / analytics can fetch
|
|
1026
|
+
the reconstructed intelligence over plain REST. All endpoints are at
|
|
1027
|
+
`https://api.evigauge.ai/v1/observ/*` and authenticated with the same
|
|
1028
|
+
`x-evigauge-api-key` + `x-evigauge-workspace-id` headers.
|
|
1029
|
+
|
|
1030
|
+
| Endpoint | What you get back |
|
|
1031
|
+
|---|---|
|
|
1032
|
+
| `GET /v1/observ/traces?workspace_id=…` | paginated trace list |
|
|
1033
|
+
| `GET /v1/observ/traces/{tid}` | full trace summary |
|
|
1034
|
+
| `GET /v1/observ/traces/{tid}/decomposition` | reconstructed decomposition tree |
|
|
1035
|
+
| `GET /v1/observ/traces/{tid}/decision-trace` | reasoning DAG |
|
|
1036
|
+
| `GET /v1/observ/traces/{tid}/sources` | sources matrix (cited / dropped / recommended-unseen, each with a live web-verification verdict) |
|
|
1037
|
+
| `GET /v1/observ/traces/{tid}/reliability` | per-claim hallucination score |
|
|
1038
|
+
| `GET /v1/observ/usage?workspace_id=…` | cost + token usage rollups |
|
|
1039
|
+
| `GET /v1/observ/drift?workspace_id=…` | daily drift signal |
|
|
1040
|
+
|
|
1041
|
+
Quick Next.js example — a server component showing the sources matrix:
|
|
1042
|
+
|
|
1043
|
+
```ts
|
|
1044
|
+
// app/traces/[tid]/sources/page.tsx
|
|
1045
|
+
export default async function Page({ params }: { params: { tid: string } }) {
|
|
1046
|
+
const res = await fetch(
|
|
1047
|
+
`https://api.evigauge.ai/v1/observ/traces/${params.tid}/sources`,
|
|
1048
|
+
{
|
|
1049
|
+
headers: {
|
|
1050
|
+
"x-evigauge-api-key": process.env.EVIGAUGE_API_KEY!,
|
|
1051
|
+
"x-evigauge-workspace-id": process.env.EVIGAUGE_WORKSPACE_ID!,
|
|
1052
|
+
},
|
|
1053
|
+
cache: "no-store",
|
|
1054
|
+
},
|
|
1055
|
+
);
|
|
1056
|
+
const sources = await res.json();
|
|
1057
|
+
return <SourcesMatrix data={sources} />;
|
|
1058
|
+
}
|
|
1059
|
+
```
|
|
1060
|
+
|
|
1061
|
+
The OpenAPI / Swagger UI for every endpoint lives at
|
|
1062
|
+
`https://api.evigauge.ai/v1/observ/docs` (and the ingest at
|
|
1063
|
+
`https://ingest.evigauge.ai/docs`) — fully brand-styled, "Try it out" works
|
|
1064
|
+
in-browser.
|
|
1065
|
+
|
|
1066
|
+
## Beyond your app — Claude Code enterprise telemetry
|
|
1067
|
+
|
|
1068
|
+
Evigauge also ingests telemetry from **Claude Code** running across your
|
|
1069
|
+
organization — **no `evigauge-trace` install required**. Claude Code emits
|
|
1070
|
+
native OpenTelemetry metrics + events; an IT admin points every install at
|
|
1071
|
+
Evigauge with a single managed-settings file (distributed via MDM / Jamf /
|
|
1072
|
+
Intune / Group Policy) plus a workspace-scoped `cck_live_…` key. You then get,
|
|
1073
|
+
per employee:
|
|
1074
|
+
|
|
1075
|
+
- session count, active time, cost (USD), input / output / cache tokens
|
|
1076
|
+
- **git commits authored in Claude Code** — SHA, message, files changed,
|
|
1077
|
+
± lines (captured by a small `PostToolUse` hook)
|
|
1078
|
+
- lines of code added / removed, edit accept/reject decisions, top files edited
|
|
1079
|
+
|
|
1080
|
+
This is a *sibling* capability to the SDK: the SDK traces **your LLM app's**
|
|
1081
|
+
reasoning, while Claude Code telemetry tracks **your team's** use of the
|
|
1082
|
+
Claude Code CLI. Both land in the same workspace and share one Read API +
|
|
1083
|
+
Swagger. Compliance is inherited from the same layer — per-workspace
|
|
1084
|
+
`capture_text` opt-in gates prompt/tool-content, every row is PII-redacted at
|
|
1085
|
+
ingest, and Read API access is RBAC-gated + audit-logged.
|
|
1086
|
+
|
|
1087
|
+
**Read API** (`workspace_viewer` role — Better-Auth session or workspace key):
|
|
1088
|
+
|
|
1089
|
+
| Endpoint | What you get back |
|
|
1090
|
+
|---|---|
|
|
1091
|
+
| `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/usage?period=day\|week\|month` | per-employee rollup: tokens / cost / commits / PRs / lines / sessions / active-time / edit decisions |
|
|
1092
|
+
| `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/commits?limit=50` | recent Claude Code-authored commits (SHA, message, files changed, ± lines) |
|
|
1093
|
+
| `GET /v1/observ/orgs/{org}/workspaces/{ws}/claude-code/top-files?period=week` | most-edited files across the workspace |
|
|
1094
|
+
|
|
1095
|
+
Full setup runbook — managed-settings template, the commit hook (`.sh` +
|
|
1096
|
+
`.ps1`), per-platform MDM rollout, and the PII / compliance posture — lives at
|
|
1097
|
+
`docs/claude-code-enterprise-setup.md` in the Evigauge repo.
|
|
1098
|
+
|
|
1099
|
+
## Semantic conventions (for non-Python languages)
|
|
1100
|
+
|
|
1101
|
+
Set these on your spans (Python SDK does it for you). The reconstruction
|
|
1102
|
+
engines key off these names.
|
|
1103
|
+
|
|
1104
|
+
| Attribute | Type | Purpose |
|
|
1105
|
+
|---|---|---|
|
|
1106
|
+
| `evigauge.ai` / `evigauge.workspace` / `evigauge.project` | resource attrs | tenancy stamp on every span |
|
|
1107
|
+
| `evigauge.reasoning_role` | string | `decomposer` / `retriever` / `synthesizer` / `decision` / `guardrail` |
|
|
1108
|
+
| `evigauge.node_type` | string | finer-grained role; free-form taxonomy |
|
|
1109
|
+
| `gen_ai.agent.name` | string | **Agent Fleet Monitor** node identity — the agent's stable display name (fallbacks: `agent.name` / `gen_ai.agent.id` / `agent.id`). See [Agent Fleet Monitor](#agent-fleet-monitor--real-time-topology--flags). |
|
|
1110
|
+
| `evigauge.sources.used` | JSON array of strings | URLs the step actually cited |
|
|
1111
|
+
| `evigauge.sources.consulted` | JSON array of strings | URLs the step looked at |
|
|
1112
|
+
| `evigauge.sources.dropped` | JSON array of strings | URLs the step considered then rejected |
|
|
1113
|
+
| `evigauge.decision.selected` | string | which option was chosen |
|
|
1114
|
+
| `evigauge.decision.rules_fired` | JSON array of strings | named rules that influenced the decision |
|
|
1115
|
+
| `evigauge.decision.scores` | JSON object `{option: score}` | per-option scoring |
|
|
1116
|
+
| `evigauge.cost.usd` | float | computed USD cost for **any** model (see [Cost for any model](#cost-for-any-model-any-provider)); optional — we derive from tokens if absent |
|
|
1117
|
+
| `evigauge.cost.model_pricing_version` | string | which price source produced the cost (`provider` / snapshot version / `litellm:<ver>` / `openrouter-live`) |
|
|
1118
|
+
| `evigauge.cost.unpriced_model` | string | set only when no price source matched — the cost is `0.0` and this flags it (never a silent zero) |
|
|
1119
|
+
| `gen_ai.system` / `gen_ai.request.model` / `gen_ai.usage.input_tokens` / `gen_ai.usage.output_tokens` | OTel GenAI conventions | LLM call metadata |
|
|
1120
|
+
|
|
1121
|
+
JSON-encode arrays/objects as strings — OTel attributes don't support nested
|
|
1122
|
+
types natively. The backend parses them on the way in.
|
|
1123
|
+
|
|
1124
|
+
## Configuration — `init()` parameters
|
|
1125
|
+
|
|
1126
|
+
| Parameter | Default | Meaning |
|
|
1127
|
+
|-----------|---------|---------|
|
|
1128
|
+
| `org_id` / `workspace_id` / `project_id` | required | Tenancy identity stamped on every span. |
|
|
1129
|
+
| `backend_url` | required | The Evigauge backend base URL. |
|
|
1130
|
+
| `api_key` | required | Workspace API key (`opx_live_…` / `opx_test_…`). |
|
|
1131
|
+
| `transport` | `"collector"` | `"collector"` = OTLP/gRPC to `collector_endpoint`. `"direct"` = HTTPS POST straight to `backend_url`, no collector, no gRPC. Also settable via `EVIGAUGE_TRANSPORT`; the argument wins. |
|
|
1132
|
+
| `collector_endpoint` | `http://localhost:4317` | OTLP collector endpoint. Ignored when `transport="direct"`. |
|
|
1133
|
+
| `wal_path` | `.evigauge-wal/spans.jsonl` | Write-ahead-log path (see Durability). |
|
|
1134
|
+
| `auto_instrument` | `True` | Patch litellm/anthropic/openai at startup. |
|
|
1135
|
+
| `fail_open` | `False` | See "fail_open scope" in the FAQ. |
|
|
1136
|
+
| `sampler_rate` | `1.0` | Fraction of traces sampled. |
|
|
1137
|
+
|
|
1138
|
+
## Durability — the write-ahead log
|
|
1139
|
+
|
|
1140
|
+
Every span is `fsync`-ed to a local WAL (`wal_path`) *before* it is handed to the
|
|
1141
|
+
exporter, so a process that crashes mid-export leaves a durable on-disk record of
|
|
1142
|
+
what was in flight. On a clean shutdown the WAL is deleted; spans that never made it
|
|
1143
|
+
out survive on disk.
|
|
1144
|
+
|
|
1145
|
+
**What the next `init()` does with that WAL, precisely:** it counts the entries, logs
|
|
1146
|
+
a warning, and renames the file to `<wal_path>.replayed-<timestamp>`. It does **not**
|
|
1147
|
+
re-export them — full OTLP round-trip replay is not implemented yet. The renamed file
|
|
1148
|
+
is there so an operator can re-emit it manually. Do not read "WAL-backed" as
|
|
1149
|
+
"guaranteed at-least-once delivery to the backend"; read it as "a crash never
|
|
1150
|
+
silently loses the evidence."
|
|
1151
|
+
|
|
1152
|
+
Past `max_wal_bytes` (10 MB default) the WAL write is refused and a counter advances.
|
|
1153
|
+
The span is still exported — only its crash-durability is lost.
|
|
1154
|
+
|
|
1155
|
+
## Troubleshooting
|
|
1156
|
+
|
|
1157
|
+
- **Spans are accepted but never appear.** The ingest API returns `202` even when it
|
|
1158
|
+
dead-letters every span in the batch — a rejected span is quarantined, not dropped,
|
|
1159
|
+
and the `202` refers to receipt, not validity. On `transport="direct"` the SDK reads
|
|
1160
|
+
the response's `partialSuccess` field and logs
|
|
1161
|
+
`N span(s) were DEAD-LETTERED by ingest`. **If you are on `0.1.0a11` or earlier,
|
|
1162
|
+
this is almost certainly you** — upgrade (see [What's new](#whats-new-in-010a12)).
|
|
1163
|
+
- **`415 OTLP/protobuf is not supported`.** Your collector is exporting protobuf. Set
|
|
1164
|
+
`encoding: json` on its `otlphttp` exporter. The SDK's `transport="direct"` path is
|
|
1165
|
+
unaffected.
|
|
1166
|
+
- **`404` on `/v1/traces`.** You are pointed at an Evigauge backend that predates native
|
|
1167
|
+
OTLP support. Upgrade the backend, or use `transport="direct"` which posts to
|
|
1168
|
+
`/v1/otlp/traces`.
|
|
1169
|
+
- **No spans appear at all.** Check `backend_url` / `collector_endpoint` reachability
|
|
1170
|
+
and that `api_key` is a live (not revoked) workspace key. A `401` is logged
|
|
1171
|
+
explicitly by the direct exporter.
|
|
1172
|
+
- **A custom `evigauge.*` attribute made spans vanish.** `EvigaugeSpanAttributes` is
|
|
1173
|
+
`extra="forbid"`: one unknown `evigauge.`-prefixed key dead-letters the whole span.
|
|
1174
|
+
Put your own attributes under any other prefix — the SDK uses `evigauge_sdk.` for its
|
|
1175
|
+
internal diagnostics for exactly this reason.
|
|
1176
|
+
- **`init() has not been called`.** A `@observe` / `ReasoningTrace` ran before
|
|
1177
|
+
`init()`. Call `init()` once at process startup, before any traced code.
|
|
1178
|
+
- **Auto-instrument patched 0 clients.** None of litellm/anthropic/openai are
|
|
1179
|
+
importable in the process — Pattern A has nothing to patch. Use Pattern B/C.
|
|
1180
|
+
- **`PermissionError` on `init()` (Windows).** Fixed in `0.1.0a12`; earlier releases
|
|
1181
|
+
renamed the WAL file while it was still open.
|
|
1182
|
+
|
|
1183
|
+
## FAQ
|
|
1184
|
+
|
|
1185
|
+
**Do I need Python to use Evigauge?** No. Ingest accepts standard **OTLP/JSON**, so any
|
|
1186
|
+
language with an OpenTelemetry SDK (TypeScript, Go, Java, .NET, …) can point a stock
|
|
1187
|
+
exporter at it and get the full reconstruction stack. The API key alone identifies
|
|
1188
|
+
your org and workspace, so a stock span needs **no** `evigauge.*` attributes at all:
|
|
1189
|
+
|
|
1190
|
+
```bash
|
|
1191
|
+
export OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.evigauge.ai
|
|
1192
|
+
export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/json # protobuf gets a 415
|
|
1193
|
+
export OTEL_EXPORTER_OTLP_HEADERS="x-evigauge-api-key=opx_live_…"
|
|
1194
|
+
```
|
|
1195
|
+
|
|
1196
|
+
Both `/v1/traces` (what an exporter appends to a base endpoint) and
|
|
1197
|
+
`/v1/otlp/traces` work. Add the semantic conventions below to unlock the engines. See
|
|
1198
|
+
[Use from TypeScript / Next.js](#use-from-typescript--nextjs).
|
|
1199
|
+
|
|
1200
|
+
**Then why install this SDK?** For `auto_instrument` (zero-code tracing of
|
|
1201
|
+
litellm/anthropic/openai), the `@observe` / `ReasoningTrace` reasoning-altitude
|
|
1202
|
+
helpers, the cost resolver, and the WAL. For transport alone, a stock OTel SDK is
|
|
1203
|
+
equivalent.
|
|
1204
|
+
|
|
1205
|
+
**Is there a native `@evigauge/trace` npm package?** Not yet — on the roadmap.
|
|
1206
|
+
For now use the raw OpenTelemetry JS SDK with the semantic conventions
|
|
1207
|
+
documented above; Vercel AI SDK users get most of it for free via
|
|
1208
|
+
`experimental_telemetry`.
|
|
1209
|
+
|
|
1210
|
+
**Does `evigauge-trace` import my agent framework?** No. The adapters are
|
|
1211
|
+
duck-typed and import nothing from the framework or from the Evigauge backend.
|
|
1212
|
+
|
|
1213
|
+
**What does `fail_open` actually cover?** `fail_open=True` wraps the **whole**
|
|
1214
|
+
`init()` body — exporter construction, TracerProvider setup, WAL replay, and
|
|
1215
|
+
auto-instrument registration. A bad `collector_endpoint`, a WAL-path I/O error,
|
|
1216
|
+
or an exporter-constructor failure all degrade to a logged exception instead of
|
|
1217
|
+
taking down your startup. `set_config()` runs before the guard, so `get_config()`
|
|
1218
|
+
still works in the degraded state (you simply get no spans). With the default
|
|
1219
|
+
`fail_open=False`, any of those failures raises.
|
|
1220
|
+
|
|
1221
|
+
**Can I emit spans during process shutdown?** No. Do not start new spans after
|
|
1222
|
+
your shutdown hook runs — a span started concurrently with the exporter
|
|
1223
|
+
shutting down can be dropped (this race exists in upstream OpenTelemetry's
|
|
1224
|
+
`BatchSpanProcessor` too). Finish traced work before tearing the process down.
|
|
1225
|
+
(Tracked as deferred note B2-N7.)
|
|
1226
|
+
|
|
1227
|
+
**Is the SDK safe in production?** Yes — span export is buffered and
|
|
1228
|
+
WAL-backed, and a traced function never fails because tracing failed:
|
|
1229
|
+
attribute-extraction errors are caught and recorded on the span, and the
|
|
1230
|
+
original return value passes through untouched.
|