continuous-intelligence-layer 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. continuous_intelligence_layer/__init__.py +28 -0
  2. continuous_intelligence_layer/_core/__init__.py +6 -0
  3. continuous_intelligence_layer/_core/exporter.py +517 -0
  4. continuous_intelligence_layer/_core/graph_exporter.py +92 -0
  5. continuous_intelligence_layer/_core/utils.py +170 -0
  6. continuous_intelligence_layer/anthropic/__init__.py +9 -0
  7. continuous_intelligence_layer/anthropic/init.py +232 -0
  8. continuous_intelligence_layer/anthropic/instrumentation.py +91 -0
  9. continuous_intelligence_layer/crewai/__init__.py +9 -0
  10. continuous_intelligence_layer/crewai/init.py +228 -0
  11. continuous_intelligence_layer/crewai/instrumentation.py +83 -0
  12. continuous_intelligence_layer/langgraph/__init__.py +12 -0
  13. continuous_intelligence_layer/langgraph/init.py +253 -0
  14. continuous_intelligence_layer/langgraph/instrumentation.py +71 -0
  15. continuous_intelligence_layer/openai/__init__.py +9 -0
  16. continuous_intelligence_layer/openai/init.py +229 -0
  17. continuous_intelligence_layer/openai/instrumentation.py +67 -0
  18. continuous_intelligence_layer-0.1.0.dist-info/METADATA +633 -0
  19. continuous_intelligence_layer-0.1.0.dist-info/RECORD +38 -0
  20. continuous_intelligence_layer-0.1.0.dist-info/WHEEL +4 -0
  21. continuous_intelligence_layer-0.1.0.dist-info/licenses/LICENSE +21 -0
  22. evaluators/__init__.py +32 -0
  23. evaluators/base_evaluator.py +181 -0
  24. evaluators/crewai_input_evaluator.py +249 -0
  25. evaluators/input_evaluator.py +121 -0
  26. evaluators/models.py +180 -0
  27. evaluators/output_evaluator.py +247 -0
  28. evaluators/runner.py +313 -0
  29. evaluators/tool_agent_evaluator.py +305 -0
  30. graph_builder/__init__.py +6 -0
  31. graph_builder/builder.py +212 -0
  32. graph_builder/models.py +159 -0
  33. graph_builder/mongo_store.py +588 -0
  34. llm_router/__init__.py +3 -0
  35. llm_router/router.py +71 -0
  36. rca_engine/__init__.py +5 -0
  37. rca_engine/incident_report.py +162 -0
  38. rca_engine/rca_engine.py +202 -0
@@ -0,0 +1,28 @@
1
+ """
2
+ Continuous Intelligence Layer SDK
3
+ ==================================
4
+
5
+ A zero-infrastructure, OpenTelemetry-based observability layer for agentic
6
+ AI systems. Each supported agent framework has its own subpackage with its
7
+ own ``init()`` — import the one matching your framework:
8
+
9
+ from continuous_intelligence_layer.langgraph import init # LangGraph / LangChain
10
+ from continuous_intelligence_layer.crewai import init # CrewAI
11
+ from continuous_intelligence_layer.openai import init # direct OpenAI SDK
12
+
13
+ Quick Start
14
+ -----------
15
+ from continuous_intelligence_layer.langgraph import init
16
+
17
+ init(api_key="...") # call ONCE, before your graph runs
18
+
19
+ graph.invoke(...) # your agent code — unchanged
20
+
21
+ Every framework's ``init()`` shares the same signature and return contract
22
+ (``{"execution_id": str, "output_path": str}``), and every framework's spans
23
+ flow through the same downstream pipeline (``graph_builder/``,
24
+ ``evaluators/``, ``rca_engine/``) regardless of which subpackage produced
25
+ them — see each subpackage's docstring for framework-specific details.
26
+ """
27
+
28
+ __version__ = "0.1.0"
@@ -0,0 +1,6 @@
1
+ """Internal, framework-agnostic building blocks shared by every framework
2
+ subpackage (``langgraph``, ``crewai``, ``openai``, ...).
3
+
4
+ Not part of the public API — import from a framework subpackage instead
5
+ (e.g. ``from continuous_intelligence_layer.langgraph import init``).
6
+ """
@@ -0,0 +1,517 @@
1
+ """
2
+ exporter.py
3
+ -----------
4
+ Custom OpenTelemetry SpanExporter — the core of the SDK.
5
+
6
+ Role in the pipeline
7
+ --------------------
8
+ OTel SDK calls export() synchronously (via SimpleSpanProcessor) for every span
9
+ that finishes anywhere in the Python process. We receive a ReadableSpan object,
10
+ extract every useful field, and append one JSON line to a JSONL file.
11
+
12
+ Why a custom exporter instead of Jaeger / Phoenix / Opik?
13
+ ----------------------------------------------------------
14
+ • No external process or server needed — zero infrastructure.
15
+ • Full control over the output schema.
16
+ • JSONL is trivially grep-able, streamable, and loadable with pandas.
17
+ • Swappable: to send spans to any backend, replace this class with an
18
+ HTTPSpanExporter that POSTs the same dict to a REST endpoint.
19
+
20
+ What OpenInference writes onto spans
21
+ -------------------------------------
22
+ OpenInference follows a published semantic convention spec:
23
+ https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md
24
+
25
+ The keys we extract are listed below under "OpenInference Attribute Keys".
26
+
27
+ What IS automatically captured
28
+ --------------------------------
29
+ ✅ LLM calls — kind=LLM, model name, prompt messages, response,
30
+ token counts (prompt / completion / total)
31
+ ✅ Agent/Chain nodes — kind=CHAIN or AGENT, input.value, output.value,
32
+ latency, parent-child hierarchy
33
+ ✅ Tool calls — kind=TOOL, tool.name, input, output, errors
34
+ ✅ Retriever calls — kind=RETRIEVER, input query, retrieved documents
35
+ ✅ Errors — exception events on the span → error field
36
+ ✅ Latency — computed from start_time / end_time nanoseconds
37
+ ✅ Trace / Span IDs — standard OTel identifiers (128-bit / 64-bit hex)
38
+ ✅ Parent-child links — parent_span_id enables call-tree reconstruction
39
+
40
+ What is NOT automatically captured (and why)
41
+ ---------------------------------------------
42
+ ⚠ Memory operations — LangChain's in-memory stores (ConversationBufferMemory
43
+ etc.) do NOT go through LangChain's callback system that
44
+ OpenInference hooks into, so no spans are emitted.
45
+ Workaround: wrap memory reads/writes in a custom tool.
46
+
47
+ ⚠ Handover semantics — OpenInference captures parent-child spans which *imply*
48
+ handovers, but there is no dedicated "handover" span kind.
49
+ The supervisor → sub-agent relationship is visible via
50
+ parent_span_id, but is not labelled "HANDOVER" explicitly.
51
+
52
+ ⚠ Agent role / name — The span name comes from LangGraph's node name
53
+ (e.g. "research_agent", "writer_agent") which is set
54
+ by the developer. We surface it as 'operation'.
55
+
56
+ ⚠ Intermediate state — LangGraph's StateGraph state dict is NOT captured
57
+ automatically. Only the final input/output of each
58
+ node that goes through the LangChain callback layer is.
59
+ """
60
+
61
+ from __future__ import annotations
62
+
63
+ import json
64
+ import re
65
+ import threading
66
+ import uuid
67
+ from datetime import datetime, timezone
68
+ from pathlib import Path
69
+ from typing import Any, Sequence
70
+
71
+ from opentelemetry.sdk.trace import ReadableSpan
72
+ from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
73
+
74
+
75
+ # ─────────────────────────────────────────────────────────────────────────────
76
+ # OpenInference Semantic Convention Attribute Keys
77
+ # Reference: https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md
78
+ # ─────────────────────────────────────────────────────────────────────────────
79
+
80
+ # Core I/O — present on every span kind
81
+ _OI_INPUT = "input.value" # serialized input to this node
82
+ _OI_OUTPUT = "output.value" # serialized output from this node
83
+
84
+ # LLM-specific
85
+ _OI_LLM_IN_MESSAGES = "llm.input_messages" # list of role+content dicts (newer OI)
86
+ _OI_LLM_OUT_MESSAGES = "llm.output_messages" # list of role+content dicts (newer OI)
87
+ _OI_LLM_PROMPTS = "llm.prompts" # legacy key (older OI versions)
88
+ _OI_LLM_RESPONSES = "llm.responses" # legacy key (older OI versions)
89
+ _OI_MODEL_NAME = "llm.model_name" # e.g. "gpt-4o-mini"
90
+ _OI_TOK_PROMPT = "llm.token_count.prompt"
91
+ _OI_TOK_COMPLETION = "llm.token_count.completion"
92
+ _OI_TOK_TOTAL = "llm.token_count.total"
93
+ _OI_TEMPERATURE = "llm.invocation_parameters" # JSON string with temperature etc.
94
+
95
+ # Tool-specific
96
+ _OI_TOOL_NAME = "tool.name"
97
+ _OI_TOOL_DESCRIPTION = "tool.description"
98
+ _OI_TOOL_PARAMETERS = "tool.parameters"
99
+
100
+ # Retriever-specific
101
+ _OI_RETRIEVAL_DOCS = "retrieval.documents" # list of retrieved documents
102
+
103
+ # Span kind — drives node_type classification
104
+ _OI_SPAN_KIND = "openinference.span.kind" # LLM | CHAIN | TOOL | RETRIEVER | AGENT
105
+
106
+ # Tool-calling decision — present directly on the LLM span itself (not a child
107
+ # span), so this is available regardless of framework or whether the
108
+ # framework's instrumentor also emits a separate TOOL-kind execution span.
109
+ # OTel attributes are flat, so OpenInference flattens nested lists into
110
+ # dotted+indexed keys, e.g. "llm.tools.0.tool.json_schema" and
111
+ # "llm.output_messages.0.message.tool_calls.0.tool_call.function.name".
112
+ _OI_TOOL_SCHEMA_RE = re.compile(r"^llm\.tools\.(\d+)\.tool\.json_schema$")
113
+ _OI_TOOL_CALL_NAME_RE = re.compile(
114
+ r"^llm\.output_messages\.(\d+)\.message\.tool_calls\.(\d+)\.tool_call\.function\.name$"
115
+ )
116
+ _OI_TOOL_CALL_ARGS_RE = re.compile(
117
+ r"^llm\.output_messages\.(\d+)\.message\.tool_calls\.(\d+)\.tool_call\.function\.arguments$"
118
+ )
119
+
120
+ # All known OI keys — anything else goes into 'metadata'
121
+ _ALL_OI_KEYS = {
122
+ _OI_INPUT, _OI_OUTPUT,
123
+ _OI_LLM_IN_MESSAGES, _OI_LLM_OUT_MESSAGES,
124
+ _OI_LLM_PROMPTS, _OI_LLM_RESPONSES,
125
+ _OI_MODEL_NAME, _OI_TOK_PROMPT, _OI_TOK_COMPLETION, _OI_TOK_TOTAL,
126
+ _OI_TEMPERATURE, _OI_TOOL_NAME, _OI_TOOL_DESCRIPTION, _OI_TOOL_PARAMETERS,
127
+ _OI_RETRIEVAL_DOCS, _OI_SPAN_KIND,
128
+ }
129
+
130
+ # Mapping from OpenInference span kind → our node_type vocabulary
131
+ _KIND_TO_NODE_TYPE: dict[str, str] = {
132
+ "LLM": "LLM",
133
+ "CHAIN": "Agent",
134
+ "TOOL": "Tool",
135
+ "RETRIEVER": "Retriever",
136
+ "AGENT": "Agent",
137
+ "MEMORY": "Memory",
138
+ "EMBEDDING": "Embedding",
139
+ "RERANKER": "Reranker",
140
+ "GUARDRAIL": "Guardrail",
141
+ }
142
+
143
+
144
+ # ─────────────────────────────────────────────────────────────────────────────
145
+ # Exporter
146
+ # ─────────────────────────────────────────────────────────────────────────────
147
+
148
+ class ExecutionSpanExporter(SpanExporter):
149
+ """
150
+ Writes every finished OTel span to a JSONL file.
151
+
152
+ Thread-safe: a threading.Lock guards file writes.
153
+
154
+ The JSONL format means:
155
+ - Each line is a valid, standalone JSON object.
156
+ - Files can be tailed in real-time: ``tail -f execution_trace.jsonl | jq .``
157
+ - Trivially loadable: ``pd.read_json("execution_trace.jsonl", lines=True)``
158
+
159
+ Swappability
160
+ ------------
161
+ To send spans to a backend instead of a file, subclass this and override
162
+ _write() to POST the record dict to an HTTP endpoint.
163
+ """
164
+
165
+ def __init__(self, output_path: str, execution_id: str | None = None, session_id: str | None = None):
166
+ """
167
+ Parameters
168
+ ----------
169
+ output_path :
170
+ Absolute or relative path to the JSONL file.
171
+ Parent directories are created if they don't exist.
172
+ execution_id :
173
+ Short identifier for this pipeline run. Stamped on every record.
174
+ session_id :
175
+ Identifier for the conversational session/thread.
176
+ """
177
+ self.output_path = Path(output_path)
178
+ self.execution_id = execution_id or str(uuid.uuid4())[:12]
179
+ self.session_id = session_id
180
+ self._lock = threading.Lock()
181
+
182
+ # Ensure the directory exists before the first write
183
+ self.output_path.parent.mkdir(parents=True, exist_ok=True)
184
+
185
+ # ── SpanExporter interface ────────────────────────────────────────────
186
+
187
+ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
188
+ """
189
+ Receive a batch of finished spans from the OTel SDK.
190
+
191
+ With SimpleSpanProcessor (which we use), this is called with exactly
192
+ ONE span per invocation, immediately when span.end() is called.
193
+ With BatchSpanProcessor it would be called in batches.
194
+
195
+ We extract every field we care about and write one JSON line per span.
196
+ We catch all exceptions internally — the agent must never crash because
197
+ of a tracing failure.
198
+ """
199
+ for span in spans:
200
+ try:
201
+ record = self._extract(span)
202
+ self._write(record)
203
+ except Exception as exc:
204
+ # Degrade gracefully — print but never propagate
205
+ print(f"[ExecutionSpanExporter] ⚠ Failed to export span "
206
+ f"'{span.name}': {type(exc).__name__}: {exc}")
207
+ return SpanExportResult.SUCCESS
208
+
209
+ def shutdown(self) -> None:
210
+ """Called when TracerProvider.shutdown() is invoked."""
211
+ pass
212
+
213
+ def force_flush(self, timeout_millis: int = 30_000) -> bool:
214
+ """All writes are synchronous, so there is nothing to flush."""
215
+ return True
216
+
217
+ # ── Span extraction ───────────────────────────────────────────────────
218
+
219
+ def _extract(self, span: ReadableSpan) -> dict:
220
+ """
221
+ Convert a raw ReadableSpan into our structured dict.
222
+
223
+ ReadableSpan fields used
224
+ ------------------------
225
+ span.context.trace_id — 128-bit int → 32-char hex
226
+ span.context.span_id — 64-bit int → 16-char hex
227
+ span.parent.span_id — parent's 64-bit int (None if root span)
228
+ span.name — the operation name set by the instrumentation
229
+ span.attributes — dict of all OTel + OpenInference attributes
230
+ span.start_time — int, nanoseconds since epoch (UTC)
231
+ span.end_time — int, nanoseconds since epoch (UTC)
232
+ span.status.status_code — StatusCode enum: OK | ERROR | UNSET
233
+ span.events — list of SpanEvent (exception events live here)
234
+ """
235
+ attrs = dict(span.attributes or {})
236
+ ctx = span.context
237
+ parent_ctx = span.parent
238
+
239
+ # ── IDs ──────────────────────────────────────────────────────────
240
+ trace_id = _fmt_trace_id(ctx.trace_id)
241
+ span_id = _fmt_span_id(ctx.span_id)
242
+ parent_id = _fmt_span_id(parent_ctx.span_id) if parent_ctx else None
243
+
244
+ # ── Timestamps & latency ─────────────────────────────────────────
245
+ start_ns = span.start_time or 0
246
+ end_ns = span.end_time or 0
247
+ latency_ms = round((end_ns - start_ns) / 1_000_000, 3) if (start_ns and end_ns) else 0.0
248
+
249
+ # ── Node type (from OpenInference span kind or name heuristic) ───
250
+ oi_kind = _get(attrs, _OI_SPAN_KIND, "")
251
+ node_type = _KIND_TO_NODE_TYPE.get(
252
+ oi_kind.upper(),
253
+ _infer_node_type_from_name(span.name),
254
+ )
255
+
256
+ # ── Status & errors ──────────────────────────────────────────────
257
+ #
258
+ # OTel stores exceptions as SpanEvents with name="exception".
259
+ # We surface the exception.message as our error field.
260
+ #
261
+ status_code = span.status.status_code.name if span.status else "UNSET"
262
+ error_msg: str | None = None
263
+ for event in span.events or []:
264
+ if event.name == "exception":
265
+ error_msg = str(event.attributes.get("exception.message", ""))
266
+ if not error_msg:
267
+ error_msg = str(event.attributes.get("exception.type", "Unknown error"))
268
+ break
269
+ if not error_msg and status_code == "ERROR":
270
+ error_msg = getattr(span.status, "description", None) or "Unknown error"
271
+
272
+ # ── LLM token usage ──────────────────────────────────────────────
273
+ tok_prompt = _get(attrs, _OI_TOK_PROMPT)
274
+ tok_comp = _get(attrs, _OI_TOK_COMPLETION)
275
+ tok_total = _get(attrs, _OI_TOK_TOTAL)
276
+ # Compute total if missing (older OI versions don't set it)
277
+ if tok_total is None and tok_prompt is not None and tok_comp is not None:
278
+ tok_total = tok_prompt + tok_comp
279
+
280
+ # ── Prompt & response: support both old and new OI key names ─────
281
+ prompt = _get(attrs, _OI_LLM_IN_MESSAGES) or _get(attrs, _OI_LLM_PROMPTS)
282
+ response = _get(attrs, _OI_LLM_OUT_MESSAGES) or _get(attrs, _OI_LLM_RESPONSES)
283
+
284
+ # ── Tool-calling decision (available tools + what was chosen) ────
285
+ available_tools = _extract_available_tools(attrs)
286
+ tool_calls = _extract_tool_calls(attrs, _get(attrs, _OI_OUTPUT))
287
+
288
+ # ── Remaining attributes → metadata ──────────────────────────────
289
+ metadata = {
290
+ k: v for k, v in attrs.items()
291
+ if k not in _ALL_OI_KEYS
292
+ and not _OI_TOOL_SCHEMA_RE.match(k)
293
+ and not _OI_TOOL_CALL_NAME_RE.match(k)
294
+ and not _OI_TOOL_CALL_ARGS_RE.match(k)
295
+ }
296
+
297
+ # The UUID is saved as project.id
298
+ project_id = str(span.resource.attributes.get("project.id", span.resource.attributes.get("service.name", ""))) if span.resource and span.resource.attributes else None
299
+
300
+ return {
301
+ # ── Run identity ──────────────────────────────────────────
302
+ "execution_id": self.execution_id,
303
+ "session_id": self.session_id,
304
+ "project_id": project_id,
305
+
306
+ # ── Span identity & hierarchy ────────────────────────────
307
+ "trace_id": trace_id,
308
+ "span_id": span_id,
309
+ "parent_span_id": parent_id, # None = root span
310
+
311
+ # ── What happened ────────────────────────────────────────
312
+ "operation": span.name, # e.g. "ChatOpenAI", "web_search", "planner"
313
+ "node_type": node_type, # LLM | Tool | Agent | Retriever | ...
314
+
315
+ # ── Timing ───────────────────────────────────────────────
316
+ "timestamp": _ns_to_iso(start_ns),
317
+ "end_timestamp": _ns_to_iso(end_ns),
318
+ "latency_ms": latency_ms,
319
+
320
+ # ── I/O (generic — present on all span kinds) ────────────
321
+ "input": _get(attrs, _OI_INPUT),
322
+ "output": _get(attrs, _OI_OUTPUT),
323
+
324
+ # ── LLM-specific ─────────────────────────────────────────
325
+ "prompt": prompt, # full message list or prompt string
326
+ "response": response, # full message list or response string
327
+ "model": _get(attrs, _OI_MODEL_NAME),
328
+ "tokens": {
329
+ "prompt": tok_prompt,
330
+ "completion": tok_comp,
331
+ "total": tok_total,
332
+ },
333
+
334
+ # ── Tool-specific ────────────────────────────────────────
335
+ "tool_name": _get(attrs, _OI_TOOL_NAME),
336
+ "tool_desc": _get(attrs, _OI_TOOL_DESCRIPTION),
337
+
338
+ # ── Tool-calling decision (populated on LLM nodes, node-local —
339
+ # doesn't require a separate child TOOL span to exist) ─────
340
+ "available_tools": available_tools,
341
+ "tool_calls": tool_calls,
342
+
343
+ # ── Retriever-specific ───────────────────────────────────
344
+ "retrieved_docs": _get(attrs, _OI_RETRIEVAL_DOCS),
345
+
346
+ # ── Status & errors ──────────────────────────────────────
347
+ "status": status_code, # OK | ERROR | UNSET
348
+ "error": error_msg,
349
+
350
+ # ── Everything else from OTel/OI attributes ──────────────
351
+ "metadata": metadata,
352
+ }
353
+
354
+ def _write(self, record: dict) -> None:
355
+ """Append one JSON line to the JSONL file. Thread-safe."""
356
+ with self._lock:
357
+ with open(self.output_path, "a", encoding="utf-8") as fh:
358
+ fh.write(json.dumps(record, default=str) + "\n")
359
+
360
+
361
+ # ─────────────────────────────────────────────────────────────────────────────
362
+ # Helpers
363
+ # ─────────────────────────────────────────────────────────────────────────────
364
+
365
+ def _fmt_trace_id(trace_id: int) -> str:
366
+ """128-bit trace ID as 32-char lowercase hex string."""
367
+ return format(trace_id, "032x")
368
+
369
+
370
+ def _fmt_span_id(span_id: int) -> str:
371
+ """64-bit span ID as 16-char lowercase hex string."""
372
+ return format(span_id, "016x")
373
+
374
+
375
+ def _ns_to_iso(ns: int) -> str | None:
376
+ """Convert nanosecond epoch timestamp to ISO-8601 UTC string."""
377
+ if not ns:
378
+ return None
379
+ return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat()
380
+
381
+
382
+ def _get(attrs: dict, key: str, default: Any = None) -> Any:
383
+ """Safely get an attribute value; returns default if absent or None."""
384
+ val = attrs.get(key)
385
+ return val if val is not None else default
386
+
387
+
388
+ def _extract_available_tools(attrs: dict) -> list[Any] | None:
389
+ """
390
+ Reconstruct the list of tool schemas offered to the model from flattened
391
+ "llm.tools.<i>.tool.json_schema" attributes. Framework-agnostic: this
392
+ comes from the model request itself, not from any framework wrapping.
393
+ """
394
+ schemas: dict[int, Any] = {}
395
+ for key, val in attrs.items():
396
+ m = _OI_TOOL_SCHEMA_RE.match(key)
397
+ if not m:
398
+ continue
399
+ idx = int(m.group(1))
400
+ if isinstance(val, str):
401
+ try:
402
+ val = json.loads(val)
403
+ except (TypeError, ValueError):
404
+ pass
405
+ schemas[idx] = val
406
+ if not schemas:
407
+ return None
408
+ return [schemas[i] for i in sorted(schemas)]
409
+
410
+
411
+ def _extract_tool_calls(attrs: dict, output_value: str | None = None) -> list[dict] | None:
412
+ """
413
+ Reconstruct the tool-call decision(s) the model actually made from
414
+ flattened "llm.output_messages.<i>.message.tool_calls.<j>.tool_call.function.*"
415
+ attributes. This lives directly on the LLM span — no separate TOOL-kind
416
+ child span is required, so it works for raw OpenAI/Anthropic SDK agents
417
+ (which never emit a tool-execution span) just as well as for LangChain.
418
+
419
+ Falls back to parsing `output_value` (the full raw response JSON, always
420
+ present on an LLM span) when the flattened attributes are absent or
421
+ incomplete. Observed in practice: for some model/param combinations
422
+ (e.g. a custom model alias + `reasoning_effort`), the instrumentor emits
423
+ `tool_call.id` but never emits `tool_call.function.name`/`.arguments` —
424
+ so `calls` below ends up empty even though the model genuinely made a
425
+ tool call and it's fully present in the raw response. Without this
426
+ fallback that tool call becomes invisible to ToolAgentEvaluator, which
427
+ gates on `node.tool_calls` being truthy — not a "no tool call happened"
428
+ case, but a "the tracing library only wrote half its notes" case.
429
+ """
430
+ calls: dict[tuple[int, int], dict] = {}
431
+ for key, val in attrs.items():
432
+ m = _OI_TOOL_CALL_NAME_RE.match(key)
433
+ if m:
434
+ idx = (int(m.group(1)), int(m.group(2)))
435
+ calls.setdefault(idx, {})["name"] = val
436
+ continue
437
+ m = _OI_TOOL_CALL_ARGS_RE.match(key)
438
+ if m:
439
+ idx = (int(m.group(1)), int(m.group(2)))
440
+ args = val
441
+ if isinstance(args, str):
442
+ try:
443
+ args = json.loads(args)
444
+ except (TypeError, ValueError):
445
+ pass
446
+ calls.setdefault(idx, {})["arguments"] = args
447
+ if calls:
448
+ return [calls[k] for k in sorted(calls)]
449
+ return _fallback_tool_calls_from_output(output_value)
450
+
451
+
452
+ def _fallback_tool_calls_from_output(output_value: str | None) -> list[dict] | None:
453
+ """
454
+ Reconstruct tool calls directly from the raw serialized LLM response,
455
+ for the small number of known provider response shapes, when the
456
+ flattened OpenInference attributes didn't carry them. Best-effort: any
457
+ parse failure or unrecognized shape just means "nothing to add", not an
458
+ error — the caller already has no tool calls from the primary path.
459
+ """
460
+ if not output_value:
461
+ return None
462
+ try:
463
+ data = json.loads(output_value)
464
+ except (TypeError, ValueError):
465
+ return None
466
+ if not isinstance(data, dict):
467
+ return None
468
+
469
+ calls: list[dict] = []
470
+
471
+ # OpenAI Chat Completions shape:
472
+ # {"choices": [{"message": {"tool_calls": [{"function": {"name", "arguments"}}]}}]}
473
+ for choice in data.get("choices") or []:
474
+ if not isinstance(choice, dict):
475
+ continue
476
+ message = choice.get("message") or {}
477
+ for tc in message.get("tool_calls") or []:
478
+ if not isinstance(tc, dict):
479
+ continue
480
+ fn = tc.get("function") or {}
481
+ name = fn.get("name")
482
+ if not name:
483
+ continue
484
+ args = fn.get("arguments")
485
+ if isinstance(args, str):
486
+ try:
487
+ args = json.loads(args)
488
+ except (TypeError, ValueError):
489
+ pass
490
+ calls.append({"name": name, "arguments": args})
491
+
492
+ # Anthropic Messages shape:
493
+ # {"content": [{"type": "tool_use", "name": ..., "input": {...}}]}
494
+ for block in data.get("content") or []:
495
+ if isinstance(block, dict) and block.get("type") == "tool_use":
496
+ calls.append({"name": block.get("name"), "arguments": block.get("input")})
497
+
498
+ return calls or None
499
+
500
+
501
+ def _infer_node_type_from_name(name: str) -> str:
502
+ """
503
+ Fallback node-type classification when OpenInference span kind is absent.
504
+ Uses simple keyword matching on the span operation name.
505
+ """
506
+ n = name.lower()
507
+ if any(kw in n for kw in ("llm", "chat", "completion", "generate", "openai", "anthropic", "gemini")):
508
+ return "LLM"
509
+ if any(kw in n for kw in ("tool", "function", "search", "api", "execute", "run_tool")):
510
+ return "Tool"
511
+ if any(kw in n for kw in ("retriev", "fetch", "vector", "embed", "document", "store")):
512
+ return "Retriever"
513
+ if any(kw in n for kw in ("memory", "buffer", "history", "remember", "recall")):
514
+ return "Memory"
515
+ if any(kw in n for kw in ("route", "router", "supervisor", "handoff", "delegate")):
516
+ return "Router"
517
+ return "Agent" # default: assume it's an agent node
@@ -0,0 +1,92 @@
1
+ """
2
+ graph_exporter.py
3
+ -----------------
4
+ An optional SpanExporter that buffers spans in memory and flushes them
5
+ as a complete ExecutionGraph to the API's /traces/ingest endpoint upon
6
+ shutdown. The API process is the only thing that talks to MongoDB directly;
7
+ agent machines only need an api_key.
8
+ """
9
+
10
+ import threading
11
+ from typing import Sequence
12
+
13
+ import requests
14
+ from opentelemetry.sdk.trace import ReadableSpan
15
+ from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
16
+
17
+ from graph_builder.builder import TraceToGraphBuilder
18
+ from .exporter import ExecutionSpanExporter
19
+
20
+
21
+ class GraphSpanExporter(SpanExporter):
22
+ """
23
+ Buffers spans during execution and POSTs them as a graph to the API
24
+ when the TracerProvider shuts down (i.e. at the end of the script).
25
+ """
26
+
27
+ def __init__(
28
+ self,
29
+ jsonl_exporter: ExecutionSpanExporter,
30
+ session_id: str | None = None,
31
+ base_url: str | None = None,
32
+ api_key: str | None = None,
33
+ project_id: str | None = None,
34
+ run_evaluations: bool = True,
35
+ ):
36
+ """
37
+ Takes an instance of ExecutionSpanExporter to reuse its _extract logic.
38
+ """
39
+ self._jsonl_exporter = jsonl_exporter
40
+ self._session_id = session_id
41
+ self._base_url = base_url
42
+ self._api_key = api_key
43
+ self._project_id = project_id
44
+ self._run_evaluations = run_evaluations
45
+ self._spans_buffer: list[dict] = []
46
+ self._lock = threading.Lock()
47
+
48
+ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
49
+ with self._lock:
50
+ for span in spans:
51
+ try:
52
+ record = self._jsonl_exporter._extract(span)
53
+ self._spans_buffer.append(record)
54
+ except Exception as exc:
55
+ print(f"[GraphSpanExporter] ⚠ Failed to extract span: {exc}")
56
+ return SpanExportResult.SUCCESS
57
+
58
+ def force_flush(self, timeout_millis: int = 30000) -> bool:
59
+ return True
60
+
61
+ def shutdown(self) -> None:
62
+ """
63
+ Build the graph from buffered spans and POST it to the API.
64
+ """
65
+ with self._lock:
66
+ if not self._spans_buffer:
67
+ return
68
+
69
+ try:
70
+ builder = TraceToGraphBuilder()
71
+ # execution_id is shared across all spans from this run
72
+ execution_id = self._spans_buffer[0].get("execution_id", "unknown")
73
+ graph = builder.build(
74
+ self._spans_buffer,
75
+ execution_id=execution_id,
76
+ session_id=self._session_id
77
+ )
78
+ graph.project_id = self._project_id or graph.project_id
79
+
80
+ res = requests.post(
81
+ f"{self._base_url}/traces/ingest",
82
+ json=graph.model_dump(mode="json"),
83
+ params={"run_evaluations": self._run_evaluations},
84
+ headers={"x-api-key": self._api_key},
85
+ timeout=30,
86
+ )
87
+ res.raise_for_status()
88
+ print(f"[GraphSpanExporter] ✓ Successfully sent graph {execution_id} to API.")
89
+ except Exception as exc:
90
+ print(f"[GraphSpanExporter] ⚠ Failed to send graph to API: {exc}")
91
+ finally:
92
+ self._spans_buffer.clear()