continuous-intelligence-layer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuous_intelligence_layer/__init__.py +28 -0
- continuous_intelligence_layer/_core/__init__.py +6 -0
- continuous_intelligence_layer/_core/exporter.py +517 -0
- continuous_intelligence_layer/_core/graph_exporter.py +92 -0
- continuous_intelligence_layer/_core/utils.py +170 -0
- continuous_intelligence_layer/anthropic/__init__.py +9 -0
- continuous_intelligence_layer/anthropic/init.py +232 -0
- continuous_intelligence_layer/anthropic/instrumentation.py +91 -0
- continuous_intelligence_layer/crewai/__init__.py +9 -0
- continuous_intelligence_layer/crewai/init.py +228 -0
- continuous_intelligence_layer/crewai/instrumentation.py +83 -0
- continuous_intelligence_layer/langgraph/__init__.py +12 -0
- continuous_intelligence_layer/langgraph/init.py +253 -0
- continuous_intelligence_layer/langgraph/instrumentation.py +71 -0
- continuous_intelligence_layer/openai/__init__.py +9 -0
- continuous_intelligence_layer/openai/init.py +229 -0
- continuous_intelligence_layer/openai/instrumentation.py +67 -0
- continuous_intelligence_layer-0.1.0.dist-info/METADATA +633 -0
- continuous_intelligence_layer-0.1.0.dist-info/RECORD +38 -0
- continuous_intelligence_layer-0.1.0.dist-info/WHEEL +4 -0
- continuous_intelligence_layer-0.1.0.dist-info/licenses/LICENSE +21 -0
- evaluators/__init__.py +32 -0
- evaluators/base_evaluator.py +181 -0
- evaluators/crewai_input_evaluator.py +249 -0
- evaluators/input_evaluator.py +121 -0
- evaluators/models.py +180 -0
- evaluators/output_evaluator.py +247 -0
- evaluators/runner.py +313 -0
- evaluators/tool_agent_evaluator.py +305 -0
- graph_builder/__init__.py +6 -0
- graph_builder/builder.py +212 -0
- graph_builder/models.py +159 -0
- graph_builder/mongo_store.py +588 -0
- llm_router/__init__.py +3 -0
- llm_router/router.py +71 -0
- rca_engine/__init__.py +5 -0
- rca_engine/incident_report.py +162 -0
- rca_engine/rca_engine.py +202 -0
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
"""
|
|
2
|
+
mongo_store.py
|
|
3
|
+
---------------
|
|
4
|
+
Persists and queries an ExecutionGraph in MongoDB.
|
|
5
|
+
|
|
6
|
+
Mongo Schema
|
|
7
|
+
------------
|
|
8
|
+
|
|
9
|
+
Collections:
|
|
10
|
+
|
|
11
|
+
executions -- one lightweight header/summary document per execution_id.
|
|
12
|
+
Holds status/latency/node_count/timestamps/snippets and the
|
|
13
|
+
materialized `edges` array (edges are small, safe to embed).
|
|
14
|
+
nodes -- one document per ExecutionNode, keyed by node_id. This is
|
|
15
|
+
deliberately NOT embedded inside the execution header doc:
|
|
16
|
+
embedding a `nodes` array in one document risks hitting
|
|
17
|
+
MongoDB's 16MB per-document BSON limit on large executions
|
|
18
|
+
(many nodes, or long LLM prompts/responses/retrieved_docs).
|
|
19
|
+
One doc per node keeps each write bounded independently.
|
|
20
|
+
evaluations -- one document per EvaluationResult, keyed by evaluation_id.
|
|
21
|
+
`node_id` is a plain indexed field (replaces the Neo4j
|
|
22
|
+
[:HAS_EVALUATION] edge).
|
|
23
|
+
rca_reports -- one document per execution_id (1:1, same as Neo4j's
|
|
24
|
+
:RCAReport node).
|
|
25
|
+
|
|
26
|
+
Idempotent: every write is an upsert keyed on a stable natural id (`_id`),
|
|
27
|
+
mirroring Neo4j's `MERGE ON node_id`.
|
|
28
|
+
|
|
29
|
+
Useful pymongo queries
|
|
30
|
+
-----------------------
|
|
31
|
+
# All LLM calls in a session with token costs
|
|
32
|
+
db.nodes.find({"session_id": "s1", "node_type": "LLM"})
|
|
33
|
+
|
|
34
|
+
# All nodes for one execution
|
|
35
|
+
db.nodes.find({"execution_id": "abc"})
|
|
36
|
+
|
|
37
|
+
# All tool failures across sessions
|
|
38
|
+
db.nodes.find({"node_type": "Tool", "status": "ERROR"})
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import json
|
|
44
|
+
import os
|
|
45
|
+
from typing import Any
|
|
46
|
+
|
|
47
|
+
from pymongo import MongoClient
|
|
48
|
+
from pymongo.server_api import ServerApi
|
|
49
|
+
|
|
50
|
+
from .models import ExecutionEdge, ExecutionGraph, ExecutionNode
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class MongoStore:
|
|
54
|
+
"""Persists and queries execution graphs in MongoDB."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, uri: str | None = None, db_name: str | None = None):
|
|
57
|
+
self.uri = uri or os.getenv("MONGODB_URI")
|
|
58
|
+
if not self.uri:
|
|
59
|
+
raise RuntimeError("MONGODB_URI environment variable is not set.")
|
|
60
|
+
self._client = MongoClient(self.uri, server_api=ServerApi("1"))
|
|
61
|
+
self._db = self._client[db_name or os.getenv("MONGODB_DB", "agentops")]
|
|
62
|
+
self._indexes_ensured = False
|
|
63
|
+
|
|
64
|
+
def close(self) -> None:
|
|
65
|
+
self._client.close()
|
|
66
|
+
|
|
67
|
+
# ── Schema ─────────────────────────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
def create_indexes(self) -> None:
|
|
70
|
+
"""Create indexes. Idempotent — safe to call repeatedly, but cheap to
|
|
71
|
+
call once (e.g. at API startup) rather than on every write."""
|
|
72
|
+
if self._indexes_ensured:
|
|
73
|
+
return
|
|
74
|
+
self._db.executions.create_index("project_id")
|
|
75
|
+
self._db.executions.create_index("session_id")
|
|
76
|
+
|
|
77
|
+
self._db.nodes.create_index("execution_id")
|
|
78
|
+
self._db.nodes.create_index("project_id")
|
|
79
|
+
self._db.nodes.create_index("session_id")
|
|
80
|
+
self._db.nodes.create_index([("project_id", 1), ("execution_id", 1)])
|
|
81
|
+
self._db.nodes.create_index([("session_id", 1), ("node_type", 1)])
|
|
82
|
+
self._db.nodes.create_index([("project_id", 1), ("eval_overall_status", 1)])
|
|
83
|
+
self._db.nodes.create_index("status")
|
|
84
|
+
|
|
85
|
+
self._db.evaluations.create_index("node_id")
|
|
86
|
+
self._db.evaluations.create_index([("execution_id", 1), ("project_id", 1)])
|
|
87
|
+
|
|
88
|
+
self._db.rca_reports.create_index([("execution_id", 1), ("project_id", 1)])
|
|
89
|
+
self._indexes_ensured = True
|
|
90
|
+
|
|
91
|
+
# ── Write ──────────────────────────────────────────────────────────────
|
|
92
|
+
|
|
93
|
+
def save_graph(self, graph: ExecutionGraph) -> None:
|
|
94
|
+
"""Persist an entire ExecutionGraph to MongoDB (idempotent)."""
|
|
95
|
+
self.create_indexes()
|
|
96
|
+
for node in graph.nodes:
|
|
97
|
+
self._upsert_node(node)
|
|
98
|
+
self._upsert_execution_header(graph)
|
|
99
|
+
|
|
100
|
+
def _upsert_node(self, node: ExecutionNode) -> None:
|
|
101
|
+
props = {
|
|
102
|
+
"node_id": node.node_id,
|
|
103
|
+
"parent_id": node.parent_id,
|
|
104
|
+
"execution_id": node.execution_id,
|
|
105
|
+
"session_id": node.session_id,
|
|
106
|
+
"project_id": node.project_id,
|
|
107
|
+
"trace_id": node.trace_id,
|
|
108
|
+
"node_type": node.node_type,
|
|
109
|
+
"name": node.name,
|
|
110
|
+
# LLM-specific
|
|
111
|
+
"model": node.model,
|
|
112
|
+
# Tool-specific
|
|
113
|
+
"tool_name": node.tool_name,
|
|
114
|
+
"tool_desc": node.tool_desc,
|
|
115
|
+
"retrieved_docs": _safe(node.retrieved_docs),
|
|
116
|
+
"available_tools": node.available_tools,
|
|
117
|
+
"tool_calls": node.tool_calls,
|
|
118
|
+
# I/O
|
|
119
|
+
"input": _safe(node.input),
|
|
120
|
+
"output": _safe(node.output),
|
|
121
|
+
"prompt": _safe(node.prompt),
|
|
122
|
+
"response": _safe(node.response),
|
|
123
|
+
# Metrics
|
|
124
|
+
"latency_ms": node.latency_ms,
|
|
125
|
+
"tokens_prompt": node.tokens.prompt,
|
|
126
|
+
"tokens_completion": node.tokens.completion,
|
|
127
|
+
"tokens_total": node.tokens.total,
|
|
128
|
+
# Status
|
|
129
|
+
"error": node.error,
|
|
130
|
+
"status": node.status,
|
|
131
|
+
# Temporal
|
|
132
|
+
"timestamp": node.timestamp,
|
|
133
|
+
"end_timestamp": node.end_timestamp,
|
|
134
|
+
# Extras
|
|
135
|
+
"metadata": json.dumps(node.metadata, default=str)[:2000],
|
|
136
|
+
}
|
|
137
|
+
self._db.nodes.update_one(
|
|
138
|
+
{"_id": node.node_id}, {"$set": props}, upsert=True
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def _upsert_execution_header(self, graph: ExecutionGraph) -> None:
|
|
142
|
+
"""Write/refresh the lightweight `executions` summary document —
|
|
143
|
+
avoids the N+1 pattern of re-fetching full node bodies just to
|
|
144
|
+
render a trace-list row."""
|
|
145
|
+
nodes = graph.nodes
|
|
146
|
+
edges = [
|
|
147
|
+
{"src": e.source_id, "rel": e.edge_type, "tgt": e.target_id}
|
|
148
|
+
for e in graph.edges
|
|
149
|
+
]
|
|
150
|
+
total_latency = sum(n.latency_ms for n in nodes)
|
|
151
|
+
status = "Success"
|
|
152
|
+
if any(n.status == "ERROR" or n.error for n in nodes):
|
|
153
|
+
status = "Failure"
|
|
154
|
+
timestamps = [n.timestamp for n in nodes if n.timestamp]
|
|
155
|
+
end_timestamps = [n.end_timestamp for n in nodes if n.end_timestamp]
|
|
156
|
+
|
|
157
|
+
targets = {e.target_id for e in graph.edges}
|
|
158
|
+
root_nodes = [n for n in nodes if n.node_id not in targets]
|
|
159
|
+
root = root_nodes[0] if root_nodes else (nodes[0] if nodes else None)
|
|
160
|
+
|
|
161
|
+
# ExecutionGraph.project_id/session_id are often left unset by the
|
|
162
|
+
# builder (it only stamps project_id onto individual nodes) — fall
|
|
163
|
+
# back to the nodes' own values so the header doc is still filterable.
|
|
164
|
+
project_id = graph.project_id or next(
|
|
165
|
+
(n.project_id for n in nodes if n.project_id), None
|
|
166
|
+
)
|
|
167
|
+
session_id = graph.session_id or next(
|
|
168
|
+
(n.session_id for n in nodes if n.session_id), None
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
header = {
|
|
172
|
+
"project_id": project_id,
|
|
173
|
+
"session_id": session_id,
|
|
174
|
+
"status": status,
|
|
175
|
+
"latency_ms": round(total_latency, 2),
|
|
176
|
+
"node_count": len(nodes),
|
|
177
|
+
"timestamp": min(timestamps) if timestamps else None,
|
|
178
|
+
"end_timestamp": max(end_timestamps) if end_timestamps else None,
|
|
179
|
+
"input_snippet": _safe(root.input) if root else None,
|
|
180
|
+
"output_snippet": _safe(root.output) if root else None,
|
|
181
|
+
"edges": edges,
|
|
182
|
+
}
|
|
183
|
+
self._db.executions.update_one(
|
|
184
|
+
{"_id": graph.execution_id}, {"$set": header}, upsert=True
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# ── Read ───────────────────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
def get_graph(self, execution_id: str, project_id: str | None = None) -> dict:
|
|
190
|
+
"""Return all nodes and edges for a given execution_id."""
|
|
191
|
+
node_filter: dict = {"execution_id": execution_id}
|
|
192
|
+
if project_id:
|
|
193
|
+
node_filter["project_id"] = project_id
|
|
194
|
+
nodes = list(self._db.nodes.find(node_filter))
|
|
195
|
+
|
|
196
|
+
header = self._db.executions.find_one({"_id": execution_id}) or {}
|
|
197
|
+
edges = header.get("edges", [])
|
|
198
|
+
|
|
199
|
+
return {"nodes": nodes, "edges": edges}
|
|
200
|
+
|
|
201
|
+
def get_session_graph(self, session_id: str) -> dict:
|
|
202
|
+
"""
|
|
203
|
+
Return all nodes and edges for an entire session (all executions).
|
|
204
|
+
A session = one user conversation thread, spanning multiple executions.
|
|
205
|
+
"""
|
|
206
|
+
nodes = list(self._db.nodes.find({"session_id": session_id}))
|
|
207
|
+
exec_ids = {n["execution_id"] for n in nodes if n.get("execution_id")}
|
|
208
|
+
edges: list[dict] = []
|
|
209
|
+
for eid in exec_ids:
|
|
210
|
+
header = self._db.executions.find_one({"_id": eid}) or {}
|
|
211
|
+
edges.extend(header.get("edges", []))
|
|
212
|
+
return {"nodes": nodes, "edges": edges}
|
|
213
|
+
|
|
214
|
+
def get_llm_spans(self, session_id: str) -> list[dict]:
|
|
215
|
+
"""Return all LLM spans for a session — for cost/token analytics."""
|
|
216
|
+
cursor = self._db.nodes.find(
|
|
217
|
+
{"session_id": session_id, "node_type": "LLM"},
|
|
218
|
+
{
|
|
219
|
+
"name": 1, "model": 1, "tokens_prompt": 1,
|
|
220
|
+
"tokens_completion": 1, "tokens_total": 1,
|
|
221
|
+
"latency_ms": 1, "execution_id": 1,
|
|
222
|
+
},
|
|
223
|
+
).sort("timestamp", 1)
|
|
224
|
+
return list(cursor)
|
|
225
|
+
|
|
226
|
+
def get_tool_spans(self, session_id: str) -> list[dict]:
|
|
227
|
+
"""Return all tool call spans for a session — for tool usage analytics."""
|
|
228
|
+
cursor = self._db.nodes.find(
|
|
229
|
+
{"session_id": session_id, "node_type": "Tool"},
|
|
230
|
+
{
|
|
231
|
+
"tool_name": 1, "input": 1, "output": 1, "latency_ms": 1,
|
|
232
|
+
"status": 1, "error": 1, "execution_id": 1,
|
|
233
|
+
},
|
|
234
|
+
).sort("timestamp", 1)
|
|
235
|
+
return list(cursor)
|
|
236
|
+
|
|
237
|
+
def get_failed_nodes(self, session_id: str | None = None) -> list[dict]:
|
|
238
|
+
"""Return all ERROR nodes, optionally scoped to a session."""
|
|
239
|
+
query: dict = {"status": "ERROR"}
|
|
240
|
+
if session_id:
|
|
241
|
+
query["session_id"] = session_id
|
|
242
|
+
cursor = self._db.nodes.find(
|
|
243
|
+
query,
|
|
244
|
+
{"name": 1, "node_type": 1, "error": 1, "execution_id": 1, "session_id": 1},
|
|
245
|
+
)
|
|
246
|
+
return list(cursor)
|
|
247
|
+
|
|
248
|
+
def get_all_execution_ids(self, project_id: str | None = None) -> list[str]:
|
|
249
|
+
query: dict = {}
|
|
250
|
+
if project_id:
|
|
251
|
+
query["project_id"] = project_id
|
|
252
|
+
return self._db.nodes.distinct("execution_id", query)
|
|
253
|
+
|
|
254
|
+
def get_all_session_ids(self) -> list[str]:
|
|
255
|
+
return self._db.nodes.distinct("session_id", {"session_id": {"$ne": None}})
|
|
256
|
+
|
|
257
|
+
# ── Trace list/detail (fixes the traces.py N+1 pattern) ────────────────
|
|
258
|
+
|
|
259
|
+
def get_trace_summaries(self, project_id: str) -> list[dict]:
|
|
260
|
+
"""One row per execution, read straight off the `executions` header
|
|
261
|
+
doc — no per-row node fetch (the N+1 the Neo4j-era code had)."""
|
|
262
|
+
cursor = self._db.executions.find({"project_id": project_id}).sort(
|
|
263
|
+
"timestamp", -1
|
|
264
|
+
)
|
|
265
|
+
return [
|
|
266
|
+
{
|
|
267
|
+
"execution_id": h["_id"],
|
|
268
|
+
"status": h.get("status", "Success"),
|
|
269
|
+
"latency": h.get("latency_ms", 0),
|
|
270
|
+
"has_evaluation": h.get("has_evaluation", False),
|
|
271
|
+
"nodes_count": h.get("node_count", 0),
|
|
272
|
+
"timestamp": h.get("timestamp"),
|
|
273
|
+
"input": h.get("input_snippet"),
|
|
274
|
+
"output": h.get("output_snippet"),
|
|
275
|
+
}
|
|
276
|
+
for h in cursor
|
|
277
|
+
]
|
|
278
|
+
|
|
279
|
+
def get_trace_detail(self, execution_id: str, project_id: str | None = None) -> dict:
|
|
280
|
+
"""Graph + evaluations + RCA report for one execution, in one call."""
|
|
281
|
+
graph = self.get_graph(execution_id, project_id=project_id)
|
|
282
|
+
|
|
283
|
+
eval_filter: dict = {"execution_id": execution_id}
|
|
284
|
+
if project_id:
|
|
285
|
+
eval_filter["project_id"] = project_id
|
|
286
|
+
evals = [
|
|
287
|
+
{"node_id": e["node_id"], "ev": e}
|
|
288
|
+
for e in self._db.evaluations.find(eval_filter)
|
|
289
|
+
]
|
|
290
|
+
|
|
291
|
+
rca_filter: dict = {"_id": execution_id}
|
|
292
|
+
if project_id:
|
|
293
|
+
rca_filter["project_id"] = project_id
|
|
294
|
+
rca = self._db.rca_reports.find_one(rca_filter)
|
|
295
|
+
|
|
296
|
+
return {"graph": graph, "evaluations": evals, "rca": rca}
|
|
297
|
+
|
|
298
|
+
def count_evaluations(self, execution_id: str, project_id: str | None = None) -> int:
|
|
299
|
+
query: dict = {"execution_id": execution_id}
|
|
300
|
+
if project_id:
|
|
301
|
+
query["project_id"] = project_id
|
|
302
|
+
return self._db.evaluations.count_documents(query)
|
|
303
|
+
|
|
304
|
+
def set_eval_status(self, execution_id: str, status: str, error: str | None = None) -> None:
|
|
305
|
+
"""Set run-level evaluation status ('running'/'done'/'error') on the
|
|
306
|
+
execution header doc, so the API can report progress independent of
|
|
307
|
+
whichever request/background task is actually doing the work."""
|
|
308
|
+
self._db.executions.update_one(
|
|
309
|
+
{"_id": execution_id},
|
|
310
|
+
{"$set": {"eval_status": status, "eval_error": error}},
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
def get_eval_status(self, execution_id: str, project_id: str | None = None) -> dict:
|
|
314
|
+
query: dict = {"_id": execution_id}
|
|
315
|
+
if project_id:
|
|
316
|
+
query["project_id"] = project_id
|
|
317
|
+
doc = self._db.executions.find_one(query, {"eval_status": 1, "eval_error": 1}) or {}
|
|
318
|
+
return {
|
|
319
|
+
"status": doc.get("eval_status", "not_started"),
|
|
320
|
+
"error": doc.get("eval_error"),
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
# ── KPI aggregation ───────────────────────────────────────────────────────
|
|
324
|
+
# All queries here are index-backed aggregation pipelines over the
|
|
325
|
+
# `nodes` collection's flat evaluation props — never a full graph fetch.
|
|
326
|
+
|
|
327
|
+
def _kpi_aggregate(self, match: dict) -> dict:
|
|
328
|
+
nodes = self._db.nodes
|
|
329
|
+
|
|
330
|
+
total = len(nodes.distinct("execution_id", match))
|
|
331
|
+
|
|
332
|
+
evaluated = len(
|
|
333
|
+
nodes.distinct(
|
|
334
|
+
"execution_id", {**match, "eval_overall_status": {"$ne": None}}
|
|
335
|
+
)
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
breakdown_rows = list(
|
|
339
|
+
nodes.aggregate(
|
|
340
|
+
[
|
|
341
|
+
{"$match": {**match, "eval_overall_status": {"$ne": None}}},
|
|
342
|
+
{
|
|
343
|
+
"$group": {
|
|
344
|
+
"_id": "$execution_id",
|
|
345
|
+
"statuses": {"$addToSet": "$eval_overall_status"},
|
|
346
|
+
}
|
|
347
|
+
},
|
|
348
|
+
{
|
|
349
|
+
"$project": {
|
|
350
|
+
"trace_status": {
|
|
351
|
+
"$switch": {
|
|
352
|
+
"branches": [
|
|
353
|
+
{
|
|
354
|
+
"case": {"$in": ["FAIL", "$statuses"]},
|
|
355
|
+
"then": "FAIL",
|
|
356
|
+
},
|
|
357
|
+
{
|
|
358
|
+
"case": {"$in": ["WARNING", "$statuses"]},
|
|
359
|
+
"then": "WARNING",
|
|
360
|
+
},
|
|
361
|
+
],
|
|
362
|
+
"default": "PASS",
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
},
|
|
367
|
+
{"$group": {"_id": "$trace_status", "c": {"$sum": 1}}},
|
|
368
|
+
]
|
|
369
|
+
)
|
|
370
|
+
)
|
|
371
|
+
breakdown = {"PASS": 0, "WARNING": 0, "FAIL": 0}
|
|
372
|
+
for row in breakdown_rows:
|
|
373
|
+
breakdown[row["_id"]] = row["c"]
|
|
374
|
+
|
|
375
|
+
tool_row = next(
|
|
376
|
+
nodes.aggregate(
|
|
377
|
+
[
|
|
378
|
+
{"$match": {**match, "tool_selection_correct": {"$ne": None}}},
|
|
379
|
+
{
|
|
380
|
+
"$group": {
|
|
381
|
+
"_id": None,
|
|
382
|
+
"total": {"$sum": 1},
|
|
383
|
+
"correct": {
|
|
384
|
+
"$sum": {"$cond": ["$tool_selection_correct", 1, 0]}
|
|
385
|
+
},
|
|
386
|
+
}
|
|
387
|
+
},
|
|
388
|
+
]
|
|
389
|
+
),
|
|
390
|
+
{"total": 0, "correct": 0},
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
hallucination = len(
|
|
394
|
+
nodes.distinct("execution_id", {**match, "has_hallucination": True})
|
|
395
|
+
)
|
|
396
|
+
injection = len(
|
|
397
|
+
nodes.distinct("execution_id", {**match, "has_injection": True})
|
|
398
|
+
)
|
|
399
|
+
critical = len(
|
|
400
|
+
nodes.distinct("execution_id", {**match, "eval_severity": "CRITICAL"})
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
latency_row = next(
|
|
404
|
+
nodes.aggregate(
|
|
405
|
+
[
|
|
406
|
+
{"$match": match},
|
|
407
|
+
{
|
|
408
|
+
"$group": {
|
|
409
|
+
"_id": "$execution_id",
|
|
410
|
+
"total_latency": {"$sum": "$latency_ms"},
|
|
411
|
+
}
|
|
412
|
+
},
|
|
413
|
+
{"$group": {"_id": None, "avg_latency": {"$avg": "$total_latency"}}},
|
|
414
|
+
]
|
|
415
|
+
),
|
|
416
|
+
None,
|
|
417
|
+
)
|
|
418
|
+
latency = latency_row["avg_latency"] if latency_row else None
|
|
419
|
+
|
|
420
|
+
tool_total = tool_row["total"] or 0
|
|
421
|
+
tool_correct = tool_row["correct"] or 0
|
|
422
|
+
|
|
423
|
+
return {
|
|
424
|
+
"total_traces": total,
|
|
425
|
+
"evaluated_traces": evaluated,
|
|
426
|
+
"unevaluated_traces": total - evaluated,
|
|
427
|
+
"trace_status_breakdown": breakdown,
|
|
428
|
+
"tool_selection_accuracy_pct": (
|
|
429
|
+
round(100 * tool_correct / tool_total, 1) if tool_total else None
|
|
430
|
+
),
|
|
431
|
+
"tool_selection_judged_count": tool_total,
|
|
432
|
+
"hallucination_count": hallucination,
|
|
433
|
+
"injection_count": injection,
|
|
434
|
+
"critical_failure_count": critical,
|
|
435
|
+
"avg_latency_ms": round(latency, 1) if latency is not None else None,
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
def get_project_kpis(self, project_id: str) -> dict:
|
|
439
|
+
"""Aggregate KPIs for a single project."""
|
|
440
|
+
return self._kpi_aggregate({"project_id": project_id})
|
|
441
|
+
|
|
442
|
+
def get_global_kpis(self, project_ids: list[str]) -> dict:
|
|
443
|
+
"""Aggregate KPIs across all of the given projects (e.g. one user's projects)."""
|
|
444
|
+
return self._kpi_aggregate({"project_id": {"$in": project_ids}})
|
|
445
|
+
|
|
446
|
+
# ── Validator Engine integration ────────────────────────────────────────
|
|
447
|
+
|
|
448
|
+
def update_node_validation(self, node_id: str, validation_results: list[dict]) -> None:
|
|
449
|
+
"""Attach validator results to a node after validation pass (legacy)."""
|
|
450
|
+
self._db.nodes.update_one(
|
|
451
|
+
{"_id": node_id},
|
|
452
|
+
{"$set": {"validation_results": json.dumps(validation_results, default=str)}},
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
def write_evaluation_result(self, result: object, project_id: str | None = None) -> None:
|
|
456
|
+
"""
|
|
457
|
+
Persist a single EvaluationResult document, keyed by evaluation_id,
|
|
458
|
+
with `node_id` as a plain indexed field (replaces Neo4j's
|
|
459
|
+
[:HAS_EVALUATION] edge). `project_id` is accepted explicitly since
|
|
460
|
+
EvaluationResult itself doesn't carry one — it's needed so
|
|
461
|
+
get_trace_detail/count_evaluations can scope by tenant without a
|
|
462
|
+
join back to the node document.
|
|
463
|
+
"""
|
|
464
|
+
props = {
|
|
465
|
+
"evaluation_id": result.evaluation_id,
|
|
466
|
+
"evaluator": result.evaluator,
|
|
467
|
+
"node_id": result.node_id,
|
|
468
|
+
"node_name": result.node_name,
|
|
469
|
+
"node_type": result.node_type,
|
|
470
|
+
"execution_id": result.execution_id,
|
|
471
|
+
"project_id": project_id,
|
|
472
|
+
"session_id": result.session_id,
|
|
473
|
+
"status": str(result.status.value if hasattr(result.status, 'value') else result.status),
|
|
474
|
+
"severity": str(result.severity.value if result.severity and hasattr(result.severity, 'value') else result.severity) if result.severity else None,
|
|
475
|
+
"confidence": result.confidence,
|
|
476
|
+
"reason": result.reason[:2000] if result.reason else "",
|
|
477
|
+
"suggestion": result.suggestion,
|
|
478
|
+
"checks": json.dumps(result.checks, default=str)[:4000],
|
|
479
|
+
"timestamp": result.timestamp,
|
|
480
|
+
"metadata": json.dumps(result.metadata or {}, default=str)[:2000],
|
|
481
|
+
}
|
|
482
|
+
self._db.evaluations.update_one(
|
|
483
|
+
{"_id": result.evaluation_id}, {"$set": props}, upsert=True
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
def update_node_evaluation(
|
|
487
|
+
self,
|
|
488
|
+
node_id: str,
|
|
489
|
+
flat_props: dict,
|
|
490
|
+
all_results: list[dict],
|
|
491
|
+
) -> None:
|
|
492
|
+
"""
|
|
493
|
+
Write flat boolean evaluation properties directly onto the node
|
|
494
|
+
document for fast filtering, and store the full results JSON blob.
|
|
495
|
+
Also denormalizes `has_evaluation`/`status` onto the execution header
|
|
496
|
+
doc (escalating Success -> Warning -> Failure, never downgrading) so
|
|
497
|
+
`get_trace_summaries` never needs a node fetch to render the list.
|
|
498
|
+
"""
|
|
499
|
+
update = dict(flat_props)
|
|
500
|
+
update["evaluation_results"] = _safe_truncate_json_list(all_results)
|
|
501
|
+
node = self._db.nodes.find_one_and_update(
|
|
502
|
+
{"_id": node_id}, {"$set": update}
|
|
503
|
+
)
|
|
504
|
+
if node and node.get("execution_id"):
|
|
505
|
+
eid = node["execution_id"]
|
|
506
|
+
overall = flat_props.get("eval_overall_status")
|
|
507
|
+
header = self._db.executions.find_one({"_id": eid}) or {}
|
|
508
|
+
priority = {"Success": 0, "Warning": 1, "Failure": 2}
|
|
509
|
+
new_status = header.get("status", "Success")
|
|
510
|
+
if overall == "FAIL":
|
|
511
|
+
new_status = "Failure"
|
|
512
|
+
elif overall == "WARNING" and priority.get(new_status, 0) < priority["Warning"]:
|
|
513
|
+
new_status = "Warning"
|
|
514
|
+
self._db.executions.update_one(
|
|
515
|
+
{"_id": eid}, {"$set": {"has_evaluation": True, "status": new_status}}
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
def write_rca_result(self, rca: dict) -> None:
|
|
519
|
+
"""Persist the RCA Result to MongoDB as an `rca_reports` document."""
|
|
520
|
+
if not rca.get("execution_id"):
|
|
521
|
+
return
|
|
522
|
+
|
|
523
|
+
props = {
|
|
524
|
+
"execution_id": rca["execution_id"],
|
|
525
|
+
"project_id": rca.get("project_id"),
|
|
526
|
+
"overall_status": rca.get("overall_status", "UNKNOWN"),
|
|
527
|
+
"root_cause": rca.get("root_cause", ""),
|
|
528
|
+
"confidence": rca.get("confidence", 0.0),
|
|
529
|
+
"propagation_chain": json.dumps(rca.get("propagation_chain", [])),
|
|
530
|
+
"contributing_factors": json.dumps(rca.get("contributing_factors", [])),
|
|
531
|
+
"evidence": json.dumps(rca.get("evidence", []), default=str)[:8000],
|
|
532
|
+
"recommendations": json.dumps(rca.get("recommendations", [])),
|
|
533
|
+
"raw_llm_analysis": rca.get("raw_llm_analysis", "")[:4000],
|
|
534
|
+
}
|
|
535
|
+
self._db.rca_reports.update_one(
|
|
536
|
+
{"_id": rca["execution_id"]}, {"$set": props}, upsert=True
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
# ── Maintenance ────────────────────────────────────────────────────────
|
|
540
|
+
|
|
541
|
+
def clear_execution(self, execution_id: str) -> None:
|
|
542
|
+
"""Delete all nodes, evaluations, RCA report and the header doc for
|
|
543
|
+
a given execution_id."""
|
|
544
|
+
with self._client.start_session() as session:
|
|
545
|
+
def _do_delete(s):
|
|
546
|
+
self._db.nodes.delete_many({"execution_id": execution_id}, session=s)
|
|
547
|
+
self._db.evaluations.delete_many({"execution_id": execution_id}, session=s)
|
|
548
|
+
self._db.rca_reports.delete_many({"_id": execution_id}, session=s)
|
|
549
|
+
self._db.executions.delete_many({"_id": execution_id}, session=s)
|
|
550
|
+
session.with_transaction(_do_delete)
|
|
551
|
+
|
|
552
|
+
def clear_session(self, session_id: str) -> None:
|
|
553
|
+
"""Delete all nodes across all executions in a session."""
|
|
554
|
+
exec_ids = self._db.nodes.distinct("execution_id", {"session_id": session_id})
|
|
555
|
+
with self._client.start_session() as session:
|
|
556
|
+
def _do_delete(s):
|
|
557
|
+
self._db.nodes.delete_many({"session_id": session_id}, session=s)
|
|
558
|
+
self._db.evaluations.delete_many({"session_id": session_id}, session=s)
|
|
559
|
+
if exec_ids:
|
|
560
|
+
self._db.rca_reports.delete_many({"_id": {"$in": exec_ids}}, session=s)
|
|
561
|
+
self._db.executions.delete_many({"_id": {"$in": exec_ids}}, session=s)
|
|
562
|
+
session.with_transaction(_do_delete)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _safe_truncate_json_list(items: list[dict], max_len: int = 16000) -> str:
|
|
566
|
+
"""
|
|
567
|
+
Serialize a list of dicts to JSON, dropping trailing items (never slicing
|
|
568
|
+
the string mid-object) so the result always parses as valid JSON on the
|
|
569
|
+
frontend, even if the full list would exceed max_len.
|
|
570
|
+
"""
|
|
571
|
+
remaining = list(items)
|
|
572
|
+
while remaining:
|
|
573
|
+
s = json.dumps(remaining, default=str)
|
|
574
|
+
if len(s) <= max_len:
|
|
575
|
+
return s
|
|
576
|
+
remaining = remaining[:-1]
|
|
577
|
+
return "[]"
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _safe(val: Any, max_len: int = 128000) -> str | None:
|
|
581
|
+
if val is None:
|
|
582
|
+
return None
|
|
583
|
+
if isinstance(val, str):
|
|
584
|
+
return val[:max_len]
|
|
585
|
+
try:
|
|
586
|
+
return json.dumps(val, default=str)[:max_len]
|
|
587
|
+
except Exception:
|
|
588
|
+
return str(val)[:max_len]
|
llm_router/__init__.py
ADDED
llm_router/router.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
router.py
|
|
3
|
+
---------
|
|
4
|
+
Dispatches evaluator/RCA LLM calls to whichever provider (openai, anthropic,
|
|
5
|
+
gemini) the user supplied a key for at ``init()`` time. Every evaluator and
|
|
6
|
+
the RCA engine talk to the LLM exclusively through this class -- nothing
|
|
7
|
+
else in this codebase should build an OpenAI/Anthropic/Gemini client of its
|
|
8
|
+
own for evaluation purposes.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
SUPPORTED_PROVIDERS = ("openai", "anthropic", "gemini")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class LLMRouter:
|
|
17
|
+
"""Wraps a single (provider, model, api_key) triple and exposes one
|
|
18
|
+
uniform ``call(prompt)`` method."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, provider: str, model: str, api_key: str):
|
|
21
|
+
if not api_key:
|
|
22
|
+
raise ValueError("LLMRouter requires a non-empty api_key.")
|
|
23
|
+
provider = (provider or "openai").lower()
|
|
24
|
+
if provider not in SUPPORTED_PROVIDERS:
|
|
25
|
+
raise ValueError(
|
|
26
|
+
f"Unsupported LLM provider: {provider!r}. "
|
|
27
|
+
f"Supported providers: {', '.join(SUPPORTED_PROVIDERS)}."
|
|
28
|
+
)
|
|
29
|
+
self.provider = provider
|
|
30
|
+
self.model = model
|
|
31
|
+
self.api_key = api_key
|
|
32
|
+
|
|
33
|
+
def call(self, prompt: str, json_mode: bool = False) -> str:
|
|
34
|
+
if self.provider == "openai":
|
|
35
|
+
return self._call_openai(prompt, json_mode)
|
|
36
|
+
if self.provider == "anthropic":
|
|
37
|
+
return self._call_anthropic(prompt, json_mode)
|
|
38
|
+
return self._call_gemini(prompt, json_mode)
|
|
39
|
+
|
|
40
|
+
def _call_openai(self, prompt: str, json_mode: bool) -> str:
|
|
41
|
+
from openai import OpenAI
|
|
42
|
+
|
|
43
|
+
client = OpenAI(api_key=self.api_key)
|
|
44
|
+
kwargs = {"response_format": {"type": "json_object"}} if json_mode else {}
|
|
45
|
+
resp = client.chat.completions.create(
|
|
46
|
+
model=self.model,
|
|
47
|
+
messages=[{"role": "user", "content": prompt}],
|
|
48
|
+
**kwargs,
|
|
49
|
+
)
|
|
50
|
+
return resp.choices[0].message.content
|
|
51
|
+
|
|
52
|
+
def _call_anthropic(self, prompt: str, json_mode: bool) -> str:
|
|
53
|
+
import anthropic
|
|
54
|
+
|
|
55
|
+
client = anthropic.Anthropic(api_key=self.api_key)
|
|
56
|
+
if json_mode:
|
|
57
|
+
prompt = f"{prompt}\n\nRespond with valid JSON only, no markdown fences."
|
|
58
|
+
resp = client.messages.create(
|
|
59
|
+
model=self.model,
|
|
60
|
+
max_tokens=4096,
|
|
61
|
+
messages=[{"role": "user", "content": prompt}],
|
|
62
|
+
)
|
|
63
|
+
return resp.content[0].text
|
|
64
|
+
|
|
65
|
+
def _call_gemini(self, prompt: str, json_mode: bool) -> str:
|
|
66
|
+
from google import genai
|
|
67
|
+
|
|
68
|
+
client = genai.Client(api_key=self.api_key)
|
|
69
|
+
config = {"response_mime_type": "application/json"} if json_mode else None
|
|
70
|
+
resp = client.models.generate_content(model=self.model, contents=prompt, config=config)
|
|
71
|
+
return resp.text
|