continuous-intelligence-layer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuous_intelligence_layer/__init__.py +28 -0
- continuous_intelligence_layer/_core/__init__.py +6 -0
- continuous_intelligence_layer/_core/exporter.py +517 -0
- continuous_intelligence_layer/_core/graph_exporter.py +92 -0
- continuous_intelligence_layer/_core/utils.py +170 -0
- continuous_intelligence_layer/anthropic/__init__.py +9 -0
- continuous_intelligence_layer/anthropic/init.py +232 -0
- continuous_intelligence_layer/anthropic/instrumentation.py +91 -0
- continuous_intelligence_layer/crewai/__init__.py +9 -0
- continuous_intelligence_layer/crewai/init.py +228 -0
- continuous_intelligence_layer/crewai/instrumentation.py +83 -0
- continuous_intelligence_layer/langgraph/__init__.py +12 -0
- continuous_intelligence_layer/langgraph/init.py +253 -0
- continuous_intelligence_layer/langgraph/instrumentation.py +71 -0
- continuous_intelligence_layer/openai/__init__.py +9 -0
- continuous_intelligence_layer/openai/init.py +229 -0
- continuous_intelligence_layer/openai/instrumentation.py +67 -0
- continuous_intelligence_layer-0.1.0.dist-info/METADATA +633 -0
- continuous_intelligence_layer-0.1.0.dist-info/RECORD +38 -0
- continuous_intelligence_layer-0.1.0.dist-info/WHEEL +4 -0
- continuous_intelligence_layer-0.1.0.dist-info/licenses/LICENSE +21 -0
- evaluators/__init__.py +32 -0
- evaluators/base_evaluator.py +181 -0
- evaluators/crewai_input_evaluator.py +249 -0
- evaluators/input_evaluator.py +121 -0
- evaluators/models.py +180 -0
- evaluators/output_evaluator.py +247 -0
- evaluators/runner.py +313 -0
- evaluators/tool_agent_evaluator.py +305 -0
- graph_builder/__init__.py +6 -0
- graph_builder/builder.py +212 -0
- graph_builder/models.py +159 -0
- graph_builder/mongo_store.py +588 -0
- llm_router/__init__.py +3 -0
- llm_router/router.py +71 -0
- rca_engine/__init__.py +5 -0
- rca_engine/incident_report.py +162 -0
- rca_engine/rca_engine.py +202 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Continuous Intelligence Layer SDK
|
|
3
|
+
==================================
|
|
4
|
+
|
|
5
|
+
A zero-infrastructure, OpenTelemetry-based observability layer for agentic
|
|
6
|
+
AI systems. Each supported agent framework has its own subpackage with its
|
|
7
|
+
own ``init()`` — import the one matching your framework:
|
|
8
|
+
|
|
9
|
+
from continuous_intelligence_layer.langgraph import init # LangGraph / LangChain
|
|
10
|
+
from continuous_intelligence_layer.crewai import init # CrewAI
|
|
11
|
+
from continuous_intelligence_layer.openai import init # direct OpenAI SDK
|
|
12
|
+
|
|
13
|
+
Quick Start
|
|
14
|
+
-----------
|
|
15
|
+
from continuous_intelligence_layer.langgraph import init
|
|
16
|
+
|
|
17
|
+
init(api_key="...") # call ONCE, before your graph runs
|
|
18
|
+
|
|
19
|
+
graph.invoke(...) # your agent code — unchanged
|
|
20
|
+
|
|
21
|
+
Every framework's ``init()`` shares the same signature and return contract
|
|
22
|
+
(``{"execution_id": str, "output_path": str}``), and every framework's spans
|
|
23
|
+
flow through the same downstream pipeline (``graph_builder/``,
|
|
24
|
+
``evaluators/``, ``rca_engine/``) regardless of which subpackage produced
|
|
25
|
+
them — see each subpackage's docstring for framework-specific details.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Internal, framework-agnostic building blocks shared by every framework
|
|
2
|
+
subpackage (``langgraph``, ``crewai``, ``openai``, ...).
|
|
3
|
+
|
|
4
|
+
Not part of the public API — import from a framework subpackage instead
|
|
5
|
+
(e.g. ``from continuous_intelligence_layer.langgraph import init``).
|
|
6
|
+
"""
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
"""
|
|
2
|
+
exporter.py
|
|
3
|
+
-----------
|
|
4
|
+
Custom OpenTelemetry SpanExporter — the core of the SDK.
|
|
5
|
+
|
|
6
|
+
Role in the pipeline
|
|
7
|
+
--------------------
|
|
8
|
+
OTel SDK calls export() synchronously (via SimpleSpanProcessor) for every span
|
|
9
|
+
that finishes anywhere in the Python process. We receive a ReadableSpan object,
|
|
10
|
+
extract every useful field, and append one JSON line to a JSONL file.
|
|
11
|
+
|
|
12
|
+
Why a custom exporter instead of Jaeger / Phoenix / Opik?
|
|
13
|
+
----------------------------------------------------------
|
|
14
|
+
• No external process or server needed — zero infrastructure.
|
|
15
|
+
• Full control over the output schema.
|
|
16
|
+
• JSONL is trivially grep-able, streamable, and loadable with pandas.
|
|
17
|
+
• Swappable: to send spans to any backend, replace this class with an
|
|
18
|
+
HTTPSpanExporter that POSTs the same dict to a REST endpoint.
|
|
19
|
+
|
|
20
|
+
What OpenInference writes onto spans
|
|
21
|
+
-------------------------------------
|
|
22
|
+
OpenInference follows a published semantic convention spec:
|
|
23
|
+
https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md
|
|
24
|
+
|
|
25
|
+
The keys we extract are listed below under "OpenInference Attribute Keys".
|
|
26
|
+
|
|
27
|
+
What IS automatically captured
|
|
28
|
+
--------------------------------
|
|
29
|
+
✅ LLM calls — kind=LLM, model name, prompt messages, response,
|
|
30
|
+
token counts (prompt / completion / total)
|
|
31
|
+
✅ Agent/Chain nodes — kind=CHAIN or AGENT, input.value, output.value,
|
|
32
|
+
latency, parent-child hierarchy
|
|
33
|
+
✅ Tool calls — kind=TOOL, tool.name, input, output, errors
|
|
34
|
+
✅ Retriever calls — kind=RETRIEVER, input query, retrieved documents
|
|
35
|
+
✅ Errors — exception events on the span → error field
|
|
36
|
+
✅ Latency — computed from start_time / end_time nanoseconds
|
|
37
|
+
✅ Trace / Span IDs — standard OTel identifiers (128-bit / 64-bit hex)
|
|
38
|
+
✅ Parent-child links — parent_span_id enables call-tree reconstruction
|
|
39
|
+
|
|
40
|
+
What is NOT automatically captured (and why)
|
|
41
|
+
---------------------------------------------
|
|
42
|
+
⚠ Memory operations — LangChain's in-memory stores (ConversationBufferMemory
|
|
43
|
+
etc.) do NOT go through LangChain's callback system that
|
|
44
|
+
OpenInference hooks into, so no spans are emitted.
|
|
45
|
+
Workaround: wrap memory reads/writes in a custom tool.
|
|
46
|
+
|
|
47
|
+
⚠ Handover semantics — OpenInference captures parent-child spans which *imply*
|
|
48
|
+
handovers, but there is no dedicated "handover" span kind.
|
|
49
|
+
The supervisor → sub-agent relationship is visible via
|
|
50
|
+
parent_span_id, but is not labelled "HANDOVER" explicitly.
|
|
51
|
+
|
|
52
|
+
⚠ Agent role / name — The span name comes from LangGraph's node name
|
|
53
|
+
(e.g. "research_agent", "writer_agent") which is set
|
|
54
|
+
by the developer. We surface it as 'operation'.
|
|
55
|
+
|
|
56
|
+
⚠ Intermediate state — LangGraph's StateGraph state dict is NOT captured
|
|
57
|
+
automatically. Only the final input/output of each
|
|
58
|
+
node that goes through the LangChain callback layer is.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
from __future__ import annotations
|
|
62
|
+
|
|
63
|
+
import json
|
|
64
|
+
import re
|
|
65
|
+
import threading
|
|
66
|
+
import uuid
|
|
67
|
+
from datetime import datetime, timezone
|
|
68
|
+
from pathlib import Path
|
|
69
|
+
from typing import Any, Sequence
|
|
70
|
+
|
|
71
|
+
from opentelemetry.sdk.trace import ReadableSpan
|
|
72
|
+
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
76
|
+
# OpenInference Semantic Convention Attribute Keys
|
|
77
|
+
# Reference: https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md
|
|
78
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
79
|
+
|
|
80
|
+
# Core I/O — present on every span kind
|
|
81
|
+
_OI_INPUT = "input.value" # serialized input to this node
|
|
82
|
+
_OI_OUTPUT = "output.value" # serialized output from this node
|
|
83
|
+
|
|
84
|
+
# LLM-specific
|
|
85
|
+
_OI_LLM_IN_MESSAGES = "llm.input_messages" # list of role+content dicts (newer OI)
|
|
86
|
+
_OI_LLM_OUT_MESSAGES = "llm.output_messages" # list of role+content dicts (newer OI)
|
|
87
|
+
_OI_LLM_PROMPTS = "llm.prompts" # legacy key (older OI versions)
|
|
88
|
+
_OI_LLM_RESPONSES = "llm.responses" # legacy key (older OI versions)
|
|
89
|
+
_OI_MODEL_NAME = "llm.model_name" # e.g. "gpt-4o-mini"
|
|
90
|
+
_OI_TOK_PROMPT = "llm.token_count.prompt"
|
|
91
|
+
_OI_TOK_COMPLETION = "llm.token_count.completion"
|
|
92
|
+
_OI_TOK_TOTAL = "llm.token_count.total"
|
|
93
|
+
_OI_TEMPERATURE = "llm.invocation_parameters" # JSON string with temperature etc.
|
|
94
|
+
|
|
95
|
+
# Tool-specific
|
|
96
|
+
_OI_TOOL_NAME = "tool.name"
|
|
97
|
+
_OI_TOOL_DESCRIPTION = "tool.description"
|
|
98
|
+
_OI_TOOL_PARAMETERS = "tool.parameters"
|
|
99
|
+
|
|
100
|
+
# Retriever-specific
|
|
101
|
+
_OI_RETRIEVAL_DOCS = "retrieval.documents" # list of retrieved documents
|
|
102
|
+
|
|
103
|
+
# Span kind — drives node_type classification
|
|
104
|
+
_OI_SPAN_KIND = "openinference.span.kind" # LLM | CHAIN | TOOL | RETRIEVER | AGENT
|
|
105
|
+
|
|
106
|
+
# Tool-calling decision — present directly on the LLM span itself (not a child
|
|
107
|
+
# span), so this is available regardless of framework or whether the
|
|
108
|
+
# framework's instrumentor also emits a separate TOOL-kind execution span.
|
|
109
|
+
# OTel attributes are flat, so OpenInference flattens nested lists into
|
|
110
|
+
# dotted+indexed keys, e.g. "llm.tools.0.tool.json_schema" and
|
|
111
|
+
# "llm.output_messages.0.message.tool_calls.0.tool_call.function.name".
|
|
112
|
+
_OI_TOOL_SCHEMA_RE = re.compile(r"^llm\.tools\.(\d+)\.tool\.json_schema$")
|
|
113
|
+
_OI_TOOL_CALL_NAME_RE = re.compile(
|
|
114
|
+
r"^llm\.output_messages\.(\d+)\.message\.tool_calls\.(\d+)\.tool_call\.function\.name$"
|
|
115
|
+
)
|
|
116
|
+
_OI_TOOL_CALL_ARGS_RE = re.compile(
|
|
117
|
+
r"^llm\.output_messages\.(\d+)\.message\.tool_calls\.(\d+)\.tool_call\.function\.arguments$"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# All known OI keys — anything else goes into 'metadata'
|
|
121
|
+
_ALL_OI_KEYS = {
|
|
122
|
+
_OI_INPUT, _OI_OUTPUT,
|
|
123
|
+
_OI_LLM_IN_MESSAGES, _OI_LLM_OUT_MESSAGES,
|
|
124
|
+
_OI_LLM_PROMPTS, _OI_LLM_RESPONSES,
|
|
125
|
+
_OI_MODEL_NAME, _OI_TOK_PROMPT, _OI_TOK_COMPLETION, _OI_TOK_TOTAL,
|
|
126
|
+
_OI_TEMPERATURE, _OI_TOOL_NAME, _OI_TOOL_DESCRIPTION, _OI_TOOL_PARAMETERS,
|
|
127
|
+
_OI_RETRIEVAL_DOCS, _OI_SPAN_KIND,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
# Mapping from OpenInference span kind → our node_type vocabulary
|
|
131
|
+
_KIND_TO_NODE_TYPE: dict[str, str] = {
|
|
132
|
+
"LLM": "LLM",
|
|
133
|
+
"CHAIN": "Agent",
|
|
134
|
+
"TOOL": "Tool",
|
|
135
|
+
"RETRIEVER": "Retriever",
|
|
136
|
+
"AGENT": "Agent",
|
|
137
|
+
"MEMORY": "Memory",
|
|
138
|
+
"EMBEDDING": "Embedding",
|
|
139
|
+
"RERANKER": "Reranker",
|
|
140
|
+
"GUARDRAIL": "Guardrail",
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
145
|
+
# Exporter
|
|
146
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
147
|
+
|
|
148
|
+
class ExecutionSpanExporter(SpanExporter):
|
|
149
|
+
"""
|
|
150
|
+
Writes every finished OTel span to a JSONL file.
|
|
151
|
+
|
|
152
|
+
Thread-safe: a threading.Lock guards file writes.
|
|
153
|
+
|
|
154
|
+
The JSONL format means:
|
|
155
|
+
- Each line is a valid, standalone JSON object.
|
|
156
|
+
- Files can be tailed in real-time: ``tail -f execution_trace.jsonl | jq .``
|
|
157
|
+
- Trivially loadable: ``pd.read_json("execution_trace.jsonl", lines=True)``
|
|
158
|
+
|
|
159
|
+
Swappability
|
|
160
|
+
------------
|
|
161
|
+
To send spans to a backend instead of a file, subclass this and override
|
|
162
|
+
_write() to POST the record dict to an HTTP endpoint.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
def __init__(self, output_path: str, execution_id: str | None = None, session_id: str | None = None):
|
|
166
|
+
"""
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
output_path :
|
|
170
|
+
Absolute or relative path to the JSONL file.
|
|
171
|
+
Parent directories are created if they don't exist.
|
|
172
|
+
execution_id :
|
|
173
|
+
Short identifier for this pipeline run. Stamped on every record.
|
|
174
|
+
session_id :
|
|
175
|
+
Identifier for the conversational session/thread.
|
|
176
|
+
"""
|
|
177
|
+
self.output_path = Path(output_path)
|
|
178
|
+
self.execution_id = execution_id or str(uuid.uuid4())[:12]
|
|
179
|
+
self.session_id = session_id
|
|
180
|
+
self._lock = threading.Lock()
|
|
181
|
+
|
|
182
|
+
# Ensure the directory exists before the first write
|
|
183
|
+
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
184
|
+
|
|
185
|
+
# ── SpanExporter interface ────────────────────────────────────────────
|
|
186
|
+
|
|
187
|
+
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
188
|
+
"""
|
|
189
|
+
Receive a batch of finished spans from the OTel SDK.
|
|
190
|
+
|
|
191
|
+
With SimpleSpanProcessor (which we use), this is called with exactly
|
|
192
|
+
ONE span per invocation, immediately when span.end() is called.
|
|
193
|
+
With BatchSpanProcessor it would be called in batches.
|
|
194
|
+
|
|
195
|
+
We extract every field we care about and write one JSON line per span.
|
|
196
|
+
We catch all exceptions internally — the agent must never crash because
|
|
197
|
+
of a tracing failure.
|
|
198
|
+
"""
|
|
199
|
+
for span in spans:
|
|
200
|
+
try:
|
|
201
|
+
record = self._extract(span)
|
|
202
|
+
self._write(record)
|
|
203
|
+
except Exception as exc:
|
|
204
|
+
# Degrade gracefully — print but never propagate
|
|
205
|
+
print(f"[ExecutionSpanExporter] ⚠ Failed to export span "
|
|
206
|
+
f"'{span.name}': {type(exc).__name__}: {exc}")
|
|
207
|
+
return SpanExportResult.SUCCESS
|
|
208
|
+
|
|
209
|
+
def shutdown(self) -> None:
|
|
210
|
+
"""Called when TracerProvider.shutdown() is invoked."""
|
|
211
|
+
pass
|
|
212
|
+
|
|
213
|
+
def force_flush(self, timeout_millis: int = 30_000) -> bool:
|
|
214
|
+
"""All writes are synchronous, so there is nothing to flush."""
|
|
215
|
+
return True
|
|
216
|
+
|
|
217
|
+
# ── Span extraction ───────────────────────────────────────────────────
|
|
218
|
+
|
|
219
|
+
def _extract(self, span: ReadableSpan) -> dict:
|
|
220
|
+
"""
|
|
221
|
+
Convert a raw ReadableSpan into our structured dict.
|
|
222
|
+
|
|
223
|
+
ReadableSpan fields used
|
|
224
|
+
------------------------
|
|
225
|
+
span.context.trace_id — 128-bit int → 32-char hex
|
|
226
|
+
span.context.span_id — 64-bit int → 16-char hex
|
|
227
|
+
span.parent.span_id — parent's 64-bit int (None if root span)
|
|
228
|
+
span.name — the operation name set by the instrumentation
|
|
229
|
+
span.attributes — dict of all OTel + OpenInference attributes
|
|
230
|
+
span.start_time — int, nanoseconds since epoch (UTC)
|
|
231
|
+
span.end_time — int, nanoseconds since epoch (UTC)
|
|
232
|
+
span.status.status_code — StatusCode enum: OK | ERROR | UNSET
|
|
233
|
+
span.events — list of SpanEvent (exception events live here)
|
|
234
|
+
"""
|
|
235
|
+
attrs = dict(span.attributes or {})
|
|
236
|
+
ctx = span.context
|
|
237
|
+
parent_ctx = span.parent
|
|
238
|
+
|
|
239
|
+
# ── IDs ──────────────────────────────────────────────────────────
|
|
240
|
+
trace_id = _fmt_trace_id(ctx.trace_id)
|
|
241
|
+
span_id = _fmt_span_id(ctx.span_id)
|
|
242
|
+
parent_id = _fmt_span_id(parent_ctx.span_id) if parent_ctx else None
|
|
243
|
+
|
|
244
|
+
# ── Timestamps & latency ─────────────────────────────────────────
|
|
245
|
+
start_ns = span.start_time or 0
|
|
246
|
+
end_ns = span.end_time or 0
|
|
247
|
+
latency_ms = round((end_ns - start_ns) / 1_000_000, 3) if (start_ns and end_ns) else 0.0
|
|
248
|
+
|
|
249
|
+
# ── Node type (from OpenInference span kind or name heuristic) ───
|
|
250
|
+
oi_kind = _get(attrs, _OI_SPAN_KIND, "")
|
|
251
|
+
node_type = _KIND_TO_NODE_TYPE.get(
|
|
252
|
+
oi_kind.upper(),
|
|
253
|
+
_infer_node_type_from_name(span.name),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
# ── Status & errors ──────────────────────────────────────────────
|
|
257
|
+
#
|
|
258
|
+
# OTel stores exceptions as SpanEvents with name="exception".
|
|
259
|
+
# We surface the exception.message as our error field.
|
|
260
|
+
#
|
|
261
|
+
status_code = span.status.status_code.name if span.status else "UNSET"
|
|
262
|
+
error_msg: str | None = None
|
|
263
|
+
for event in span.events or []:
|
|
264
|
+
if event.name == "exception":
|
|
265
|
+
error_msg = str(event.attributes.get("exception.message", ""))
|
|
266
|
+
if not error_msg:
|
|
267
|
+
error_msg = str(event.attributes.get("exception.type", "Unknown error"))
|
|
268
|
+
break
|
|
269
|
+
if not error_msg and status_code == "ERROR":
|
|
270
|
+
error_msg = getattr(span.status, "description", None) or "Unknown error"
|
|
271
|
+
|
|
272
|
+
# ── LLM token usage ──────────────────────────────────────────────
|
|
273
|
+
tok_prompt = _get(attrs, _OI_TOK_PROMPT)
|
|
274
|
+
tok_comp = _get(attrs, _OI_TOK_COMPLETION)
|
|
275
|
+
tok_total = _get(attrs, _OI_TOK_TOTAL)
|
|
276
|
+
# Compute total if missing (older OI versions don't set it)
|
|
277
|
+
if tok_total is None and tok_prompt is not None and tok_comp is not None:
|
|
278
|
+
tok_total = tok_prompt + tok_comp
|
|
279
|
+
|
|
280
|
+
# ── Prompt & response: support both old and new OI key names ─────
|
|
281
|
+
prompt = _get(attrs, _OI_LLM_IN_MESSAGES) or _get(attrs, _OI_LLM_PROMPTS)
|
|
282
|
+
response = _get(attrs, _OI_LLM_OUT_MESSAGES) or _get(attrs, _OI_LLM_RESPONSES)
|
|
283
|
+
|
|
284
|
+
# ── Tool-calling decision (available tools + what was chosen) ────
|
|
285
|
+
available_tools = _extract_available_tools(attrs)
|
|
286
|
+
tool_calls = _extract_tool_calls(attrs, _get(attrs, _OI_OUTPUT))
|
|
287
|
+
|
|
288
|
+
# ── Remaining attributes → metadata ──────────────────────────────
|
|
289
|
+
metadata = {
|
|
290
|
+
k: v for k, v in attrs.items()
|
|
291
|
+
if k not in _ALL_OI_KEYS
|
|
292
|
+
and not _OI_TOOL_SCHEMA_RE.match(k)
|
|
293
|
+
and not _OI_TOOL_CALL_NAME_RE.match(k)
|
|
294
|
+
and not _OI_TOOL_CALL_ARGS_RE.match(k)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
# The UUID is saved as project.id
|
|
298
|
+
project_id = str(span.resource.attributes.get("project.id", span.resource.attributes.get("service.name", ""))) if span.resource and span.resource.attributes else None
|
|
299
|
+
|
|
300
|
+
return {
|
|
301
|
+
# ── Run identity ──────────────────────────────────────────
|
|
302
|
+
"execution_id": self.execution_id,
|
|
303
|
+
"session_id": self.session_id,
|
|
304
|
+
"project_id": project_id,
|
|
305
|
+
|
|
306
|
+
# ── Span identity & hierarchy ────────────────────────────
|
|
307
|
+
"trace_id": trace_id,
|
|
308
|
+
"span_id": span_id,
|
|
309
|
+
"parent_span_id": parent_id, # None = root span
|
|
310
|
+
|
|
311
|
+
# ── What happened ────────────────────────────────────────
|
|
312
|
+
"operation": span.name, # e.g. "ChatOpenAI", "web_search", "planner"
|
|
313
|
+
"node_type": node_type, # LLM | Tool | Agent | Retriever | ...
|
|
314
|
+
|
|
315
|
+
# ── Timing ───────────────────────────────────────────────
|
|
316
|
+
"timestamp": _ns_to_iso(start_ns),
|
|
317
|
+
"end_timestamp": _ns_to_iso(end_ns),
|
|
318
|
+
"latency_ms": latency_ms,
|
|
319
|
+
|
|
320
|
+
# ── I/O (generic — present on all span kinds) ────────────
|
|
321
|
+
"input": _get(attrs, _OI_INPUT),
|
|
322
|
+
"output": _get(attrs, _OI_OUTPUT),
|
|
323
|
+
|
|
324
|
+
# ── LLM-specific ─────────────────────────────────────────
|
|
325
|
+
"prompt": prompt, # full message list or prompt string
|
|
326
|
+
"response": response, # full message list or response string
|
|
327
|
+
"model": _get(attrs, _OI_MODEL_NAME),
|
|
328
|
+
"tokens": {
|
|
329
|
+
"prompt": tok_prompt,
|
|
330
|
+
"completion": tok_comp,
|
|
331
|
+
"total": tok_total,
|
|
332
|
+
},
|
|
333
|
+
|
|
334
|
+
# ── Tool-specific ────────────────────────────────────────
|
|
335
|
+
"tool_name": _get(attrs, _OI_TOOL_NAME),
|
|
336
|
+
"tool_desc": _get(attrs, _OI_TOOL_DESCRIPTION),
|
|
337
|
+
|
|
338
|
+
# ── Tool-calling decision (populated on LLM nodes, node-local —
|
|
339
|
+
# doesn't require a separate child TOOL span to exist) ─────
|
|
340
|
+
"available_tools": available_tools,
|
|
341
|
+
"tool_calls": tool_calls,
|
|
342
|
+
|
|
343
|
+
# ── Retriever-specific ───────────────────────────────────
|
|
344
|
+
"retrieved_docs": _get(attrs, _OI_RETRIEVAL_DOCS),
|
|
345
|
+
|
|
346
|
+
# ── Status & errors ──────────────────────────────────────
|
|
347
|
+
"status": status_code, # OK | ERROR | UNSET
|
|
348
|
+
"error": error_msg,
|
|
349
|
+
|
|
350
|
+
# ── Everything else from OTel/OI attributes ──────────────
|
|
351
|
+
"metadata": metadata,
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
def _write(self, record: dict) -> None:
|
|
355
|
+
"""Append one JSON line to the JSONL file. Thread-safe."""
|
|
356
|
+
with self._lock:
|
|
357
|
+
with open(self.output_path, "a", encoding="utf-8") as fh:
|
|
358
|
+
fh.write(json.dumps(record, default=str) + "\n")
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
362
|
+
# Helpers
|
|
363
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
364
|
+
|
|
365
|
+
def _fmt_trace_id(trace_id: int) -> str:
|
|
366
|
+
"""128-bit trace ID as 32-char lowercase hex string."""
|
|
367
|
+
return format(trace_id, "032x")
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _fmt_span_id(span_id: int) -> str:
|
|
371
|
+
"""64-bit span ID as 16-char lowercase hex string."""
|
|
372
|
+
return format(span_id, "016x")
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _ns_to_iso(ns: int) -> str | None:
|
|
376
|
+
"""Convert nanosecond epoch timestamp to ISO-8601 UTC string."""
|
|
377
|
+
if not ns:
|
|
378
|
+
return None
|
|
379
|
+
return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat()
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _get(attrs: dict, key: str, default: Any = None) -> Any:
|
|
383
|
+
"""Safely get an attribute value; returns default if absent or None."""
|
|
384
|
+
val = attrs.get(key)
|
|
385
|
+
return val if val is not None else default
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _extract_available_tools(attrs: dict) -> list[Any] | None:
|
|
389
|
+
"""
|
|
390
|
+
Reconstruct the list of tool schemas offered to the model from flattened
|
|
391
|
+
"llm.tools.<i>.tool.json_schema" attributes. Framework-agnostic: this
|
|
392
|
+
comes from the model request itself, not from any framework wrapping.
|
|
393
|
+
"""
|
|
394
|
+
schemas: dict[int, Any] = {}
|
|
395
|
+
for key, val in attrs.items():
|
|
396
|
+
m = _OI_TOOL_SCHEMA_RE.match(key)
|
|
397
|
+
if not m:
|
|
398
|
+
continue
|
|
399
|
+
idx = int(m.group(1))
|
|
400
|
+
if isinstance(val, str):
|
|
401
|
+
try:
|
|
402
|
+
val = json.loads(val)
|
|
403
|
+
except (TypeError, ValueError):
|
|
404
|
+
pass
|
|
405
|
+
schemas[idx] = val
|
|
406
|
+
if not schemas:
|
|
407
|
+
return None
|
|
408
|
+
return [schemas[i] for i in sorted(schemas)]
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _extract_tool_calls(attrs: dict, output_value: str | None = None) -> list[dict] | None:
|
|
412
|
+
"""
|
|
413
|
+
Reconstruct the tool-call decision(s) the model actually made from
|
|
414
|
+
flattened "llm.output_messages.<i>.message.tool_calls.<j>.tool_call.function.*"
|
|
415
|
+
attributes. This lives directly on the LLM span — no separate TOOL-kind
|
|
416
|
+
child span is required, so it works for raw OpenAI/Anthropic SDK agents
|
|
417
|
+
(which never emit a tool-execution span) just as well as for LangChain.
|
|
418
|
+
|
|
419
|
+
Falls back to parsing `output_value` (the full raw response JSON, always
|
|
420
|
+
present on an LLM span) when the flattened attributes are absent or
|
|
421
|
+
incomplete. Observed in practice: for some model/param combinations
|
|
422
|
+
(e.g. a custom model alias + `reasoning_effort`), the instrumentor emits
|
|
423
|
+
`tool_call.id` but never emits `tool_call.function.name`/`.arguments` —
|
|
424
|
+
so `calls` below ends up empty even though the model genuinely made a
|
|
425
|
+
tool call and it's fully present in the raw response. Without this
|
|
426
|
+
fallback that tool call becomes invisible to ToolAgentEvaluator, which
|
|
427
|
+
gates on `node.tool_calls` being truthy — not a "no tool call happened"
|
|
428
|
+
case, but a "the tracing library only wrote half its notes" case.
|
|
429
|
+
"""
|
|
430
|
+
calls: dict[tuple[int, int], dict] = {}
|
|
431
|
+
for key, val in attrs.items():
|
|
432
|
+
m = _OI_TOOL_CALL_NAME_RE.match(key)
|
|
433
|
+
if m:
|
|
434
|
+
idx = (int(m.group(1)), int(m.group(2)))
|
|
435
|
+
calls.setdefault(idx, {})["name"] = val
|
|
436
|
+
continue
|
|
437
|
+
m = _OI_TOOL_CALL_ARGS_RE.match(key)
|
|
438
|
+
if m:
|
|
439
|
+
idx = (int(m.group(1)), int(m.group(2)))
|
|
440
|
+
args = val
|
|
441
|
+
if isinstance(args, str):
|
|
442
|
+
try:
|
|
443
|
+
args = json.loads(args)
|
|
444
|
+
except (TypeError, ValueError):
|
|
445
|
+
pass
|
|
446
|
+
calls.setdefault(idx, {})["arguments"] = args
|
|
447
|
+
if calls:
|
|
448
|
+
return [calls[k] for k in sorted(calls)]
|
|
449
|
+
return _fallback_tool_calls_from_output(output_value)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _fallback_tool_calls_from_output(output_value: str | None) -> list[dict] | None:
|
|
453
|
+
"""
|
|
454
|
+
Reconstruct tool calls directly from the raw serialized LLM response,
|
|
455
|
+
for the small number of known provider response shapes, when the
|
|
456
|
+
flattened OpenInference attributes didn't carry them. Best-effort: any
|
|
457
|
+
parse failure or unrecognized shape just means "nothing to add", not an
|
|
458
|
+
error — the caller already has no tool calls from the primary path.
|
|
459
|
+
"""
|
|
460
|
+
if not output_value:
|
|
461
|
+
return None
|
|
462
|
+
try:
|
|
463
|
+
data = json.loads(output_value)
|
|
464
|
+
except (TypeError, ValueError):
|
|
465
|
+
return None
|
|
466
|
+
if not isinstance(data, dict):
|
|
467
|
+
return None
|
|
468
|
+
|
|
469
|
+
calls: list[dict] = []
|
|
470
|
+
|
|
471
|
+
# OpenAI Chat Completions shape:
|
|
472
|
+
# {"choices": [{"message": {"tool_calls": [{"function": {"name", "arguments"}}]}}]}
|
|
473
|
+
for choice in data.get("choices") or []:
|
|
474
|
+
if not isinstance(choice, dict):
|
|
475
|
+
continue
|
|
476
|
+
message = choice.get("message") or {}
|
|
477
|
+
for tc in message.get("tool_calls") or []:
|
|
478
|
+
if not isinstance(tc, dict):
|
|
479
|
+
continue
|
|
480
|
+
fn = tc.get("function") or {}
|
|
481
|
+
name = fn.get("name")
|
|
482
|
+
if not name:
|
|
483
|
+
continue
|
|
484
|
+
args = fn.get("arguments")
|
|
485
|
+
if isinstance(args, str):
|
|
486
|
+
try:
|
|
487
|
+
args = json.loads(args)
|
|
488
|
+
except (TypeError, ValueError):
|
|
489
|
+
pass
|
|
490
|
+
calls.append({"name": name, "arguments": args})
|
|
491
|
+
|
|
492
|
+
# Anthropic Messages shape:
|
|
493
|
+
# {"content": [{"type": "tool_use", "name": ..., "input": {...}}]}
|
|
494
|
+
for block in data.get("content") or []:
|
|
495
|
+
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
496
|
+
calls.append({"name": block.get("name"), "arguments": block.get("input")})
|
|
497
|
+
|
|
498
|
+
return calls or None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _infer_node_type_from_name(name: str) -> str:
|
|
502
|
+
"""
|
|
503
|
+
Fallback node-type classification when OpenInference span kind is absent.
|
|
504
|
+
Uses simple keyword matching on the span operation name.
|
|
505
|
+
"""
|
|
506
|
+
n = name.lower()
|
|
507
|
+
if any(kw in n for kw in ("llm", "chat", "completion", "generate", "openai", "anthropic", "gemini")):
|
|
508
|
+
return "LLM"
|
|
509
|
+
if any(kw in n for kw in ("tool", "function", "search", "api", "execute", "run_tool")):
|
|
510
|
+
return "Tool"
|
|
511
|
+
if any(kw in n for kw in ("retriev", "fetch", "vector", "embed", "document", "store")):
|
|
512
|
+
return "Retriever"
|
|
513
|
+
if any(kw in n for kw in ("memory", "buffer", "history", "remember", "recall")):
|
|
514
|
+
return "Memory"
|
|
515
|
+
if any(kw in n for kw in ("route", "router", "supervisor", "handoff", "delegate")):
|
|
516
|
+
return "Router"
|
|
517
|
+
return "Agent" # default: assume it's an agent node
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""
|
|
2
|
+
graph_exporter.py
|
|
3
|
+
-----------------
|
|
4
|
+
An optional SpanExporter that buffers spans in memory and flushes them
|
|
5
|
+
as a complete ExecutionGraph to the API's /traces/ingest endpoint upon
|
|
6
|
+
shutdown. The API process is the only thing that talks to MongoDB directly;
|
|
7
|
+
agent machines only need an api_key.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
from typing import Sequence
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
from opentelemetry.sdk.trace import ReadableSpan
|
|
15
|
+
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
16
|
+
|
|
17
|
+
from graph_builder.builder import TraceToGraphBuilder
|
|
18
|
+
from .exporter import ExecutionSpanExporter
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class GraphSpanExporter(SpanExporter):
|
|
22
|
+
"""
|
|
23
|
+
Buffers spans during execution and POSTs them as a graph to the API
|
|
24
|
+
when the TracerProvider shuts down (i.e. at the end of the script).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
jsonl_exporter: ExecutionSpanExporter,
|
|
30
|
+
session_id: str | None = None,
|
|
31
|
+
base_url: str | None = None,
|
|
32
|
+
api_key: str | None = None,
|
|
33
|
+
project_id: str | None = None,
|
|
34
|
+
run_evaluations: bool = True,
|
|
35
|
+
):
|
|
36
|
+
"""
|
|
37
|
+
Takes an instance of ExecutionSpanExporter to reuse its _extract logic.
|
|
38
|
+
"""
|
|
39
|
+
self._jsonl_exporter = jsonl_exporter
|
|
40
|
+
self._session_id = session_id
|
|
41
|
+
self._base_url = base_url
|
|
42
|
+
self._api_key = api_key
|
|
43
|
+
self._project_id = project_id
|
|
44
|
+
self._run_evaluations = run_evaluations
|
|
45
|
+
self._spans_buffer: list[dict] = []
|
|
46
|
+
self._lock = threading.Lock()
|
|
47
|
+
|
|
48
|
+
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
49
|
+
with self._lock:
|
|
50
|
+
for span in spans:
|
|
51
|
+
try:
|
|
52
|
+
record = self._jsonl_exporter._extract(span)
|
|
53
|
+
self._spans_buffer.append(record)
|
|
54
|
+
except Exception as exc:
|
|
55
|
+
print(f"[GraphSpanExporter] ⚠ Failed to extract span: {exc}")
|
|
56
|
+
return SpanExportResult.SUCCESS
|
|
57
|
+
|
|
58
|
+
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
def shutdown(self) -> None:
|
|
62
|
+
"""
|
|
63
|
+
Build the graph from buffered spans and POST it to the API.
|
|
64
|
+
"""
|
|
65
|
+
with self._lock:
|
|
66
|
+
if not self._spans_buffer:
|
|
67
|
+
return
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
builder = TraceToGraphBuilder()
|
|
71
|
+
# execution_id is shared across all spans from this run
|
|
72
|
+
execution_id = self._spans_buffer[0].get("execution_id", "unknown")
|
|
73
|
+
graph = builder.build(
|
|
74
|
+
self._spans_buffer,
|
|
75
|
+
execution_id=execution_id,
|
|
76
|
+
session_id=self._session_id
|
|
77
|
+
)
|
|
78
|
+
graph.project_id = self._project_id or graph.project_id
|
|
79
|
+
|
|
80
|
+
res = requests.post(
|
|
81
|
+
f"{self._base_url}/traces/ingest",
|
|
82
|
+
json=graph.model_dump(mode="json"),
|
|
83
|
+
params={"run_evaluations": self._run_evaluations},
|
|
84
|
+
headers={"x-api-key": self._api_key},
|
|
85
|
+
timeout=30,
|
|
86
|
+
)
|
|
87
|
+
res.raise_for_status()
|
|
88
|
+
print(f"[GraphSpanExporter] ✓ Successfully sent graph {execution_id} to API.")
|
|
89
|
+
except Exception as exc:
|
|
90
|
+
print(f"[GraphSpanExporter] ⚠ Failed to send graph to API: {exc}")
|
|
91
|
+
finally:
|
|
92
|
+
self._spans_buffer.clear()
|