continuous-intelligence-layer 0.1.0__py3-none-any.whl

evaluators/runner.py

@@ -0,0 +1,313 @@
+"""
+runner.py
+---------
+EvaluationRunner — Orchestrates all 3 evaluators for a given execution.
+
+Usage:
+    from evaluators.runner import EvaluationRunner
+
+    report = EvaluationRunner(
+        execution_id="abc123",
+        llm_config={"provider": "openai", "model": "gpt-5.6-luna", "api_key": "..."},
+    ).run()
+    print(report.model_dump_json(indent=2))
+
+What it does:
+1. Loads the ExecutionGraph from MongoDB.
+2. Runs InputEvaluator, OutputEvaluator, ToolAgentEvaluator on every node.
+3. Writes EvaluationResults to MongoDB (`evaluations` docs + flat props on the node document).
+4. Appends all results to evaluation_results.jsonl.
+5. Triggers the RCA Engine and returns the full RCAReport.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _safe_json_loads(s: str) -> dict:
+    """Parse JSON string, returning {} on any error (handles truncated blobs)."""
+    try:
+        return json.loads(s)
+    except Exception:
+        return {}
+
+from graph_builder.models import ExecutionGraph, ExecutionNode
+from graph_builder.mongo_store import MongoStore
+from graph_builder.builder import TraceToGraphBuilder
+
+from .input_evaluator import InputEvaluator
+from .crewai_input_evaluator import CrewAIInputEvaluator
+from .output_evaluator import OutputEvaluator
+from .tool_agent_evaluator import ToolAgentEvaluator
+from .models import EvaluationResult, NodeEvaluationSummary
+from llm_router import LLMRouter
+
+
+class EvaluationRunner:
+    """
+    Runs all 3 evaluators against every node of a given execution and
+    persists results to MongoDB + JSONL.
+    """
+
+    def __init__(
+        self,
+        execution_id: str,
+        project_id: str | None = None,
+        output_path: str | None = None,
+        verbose: bool = True,
+        llm_config: dict | None = None,
+    ):
+        self.execution_id = execution_id
+        self.project_id = project_id
+        self.output_path = Path(
+            output_path or os.getenv("EVALUATION_OUTPUT_PATH", "./evaluation_results.jsonl")
+        )
+        self.verbose = verbose
+        self.llm_config = llm_config
+
+        # Storage
+        self._store = MongoStore()
+
+    def run(self) -> dict:
+        """
+        Run the full evaluation pipeline for this execution_id.
+        Returns a summary dict with all results + RCA report.
+        """
+        self._store.set_eval_status(self.execution_id, "running")
+        try:
+            if not self.llm_config:
+                raise ValueError(
+                    f"No evaluation API key available for execution_id="
+                    f"'{self.execution_id}'. Pass evaluation_api_key to init(), or "
+                    f"supply a provider/model/api_key when triggering evaluations."
+                )
+            router = LLMRouter(**self.llm_config)
+            self._input_ev        = InputEvaluator(router)
+            self._crewai_input_ev = CrewAIInputEvaluator(router)
+            self._output_ev       = OutputEvaluator(router)
+            self._tool_ev         = ToolAgentEvaluator(router)
+
+            # ── 1. Load graph ────────────────────────────────────────────
+            graph = self._load_graph()
+            if not graph or not graph.nodes:
+                raise ValueError(f"No graph found in MongoDB for execution_id='{self.execution_id}'")
+
+            if self.verbose:
+                print(f"\n[EvaluationRunner] ▶ Starting evaluation for execution: {self.execution_id}")
+                print(f"[EvaluationRunner]   Nodes to evaluate: {len(graph.nodes)}")
+
+            # ── 2. Run evaluators on every node ──────────────────────────
+            all_results: list[EvaluationResult] = []
+            summaries: list[NodeEvaluationSummary] = []
+
+            for i, node in enumerate(graph.nodes, 1):
+                if self.verbose:
+                    print(f"\n[EvaluationRunner]   [{i}/{len(graph.nodes)}] Evaluating: "
+                          f"{node.name} ({node.node_type})")
+
+                # CrewAI's Task._execute_core / lifecycle nodes never carry
+                # the resolved task text in their own input by design (see
+                # evaluators/crewai_input_evaluator.py) — for those specific
+                # node shapes, the CrewAI-specific evaluator fills the
+                # input_result slot instead of the generic InputEvaluator.
+                if self._crewai_input_ev.should_run(node, graph):
+                    input_res = self._crewai_input_ev.evaluate(node, graph)
+                else:
+                    input_res = self._input_ev.evaluate(node, graph)
+                output_res = self._output_ev.evaluate(node, graph)
+                tool_res   = self._tool_ev.evaluate(node, graph)
+
+                if self.verbose:
+                    _print_result("  Input  ", input_res)
+                    _print_result("  Output ", output_res)
+                    _print_result("  Tool   ", tool_res)
+
+                all_results.extend([input_res, output_res, tool_res])
+
+                summary = NodeEvaluationSummary(
+                    node_id=node.node_id,
+                    node_name=node.name,
+                    execution_id=self.execution_id,
+                    node_type=node.node_type,
+                    input_result=input_res,
+                    output_result=output_res,
+                    tool_agent_result=tool_res,
+                )
+                summaries.append(summary)
+
+                # ── 3. Persist to MongoDB ────────────────────────────────
+                self._persist_to_mongo(node, summary, [input_res, output_res, tool_res])
+
+            # ── 4. Write to JSONL ────────────────────────────────────────
+            self._write_jsonl(all_results)
+
+            # ── 5. Run RCA Engine ────────────────────────────────────────
+            rca_report = self._run_rca(graph, summaries, router)
+            self._store.write_rca_result(rca_report)
+
+            # ── 6. Build summary ─────────────────────────────────────────
+            fail_count = sum(1 for s in summaries if s.overall_status == "FAIL")
+            warn_count = sum(1 for s in summaries if s.overall_status == "WARNING")
+
+            if self.verbose:
+                print(f"\n[EvaluationRunner] ✅ Evaluation complete")
+                print(f"  Nodes: {len(graph.nodes)}  |  FAIL: {fail_count}  |  WARNING: {warn_count}")
+                print(f"  Results saved to: {self.output_path}")
+                print(f"  RCA Overall: {rca_report.get('overall_status', 'N/A')}")
+
+            self._store.set_eval_status(self.execution_id, "done")
+
+            return {
+                "execution_id":     self.execution_id,
+                "nodes_evaluated":  len(graph.nodes),
+                "nodes_failed":     fail_count,
+                "nodes_warned":     warn_count,
+                "evaluation_results": [r.to_dict() for r in all_results],
+                "rca_report":       rca_report,
+            }
+        except Exception as exc:
+            self._store.set_eval_status(self.execution_id, "error", error=str(exc)[:2000])
+            raise
+        finally:
+            self._store.close()
+
+    # ── Private helpers ─────────────────────────────────────────────────────
+
+    def _load_graph(self) -> ExecutionGraph:
+        """
+        Load spans from MongoDB and reconstruct an ExecutionGraph.
+        We load the raw node data and rebuild node objects.
+        """
+        raw = self._store.get_graph(self.execution_id, project_id=self.project_id)
+        raw_nodes = raw.get("nodes", [])
+        raw_edges = raw.get("edges", [])
+
+        if not raw_nodes:
+            return ExecutionGraph(execution_id=self.execution_id)
+
+        # Reconstruct ExecutionNode objects from MongoDB documents
+        from graph_builder.models import ExecutionNode, ExecutionEdge, EdgeType, TokenUsage, NodeType
+
+        nodes = []
+        for rn in raw_nodes:
+            props = rn
+            tokens = TokenUsage(
+                prompt=props.get("tokens_prompt"),
+                completion=props.get("tokens_completion"),
+                total=props.get("tokens_total"),
+            )
+            node = ExecutionNode(
+                node_id=props.get("node_id") or props.get("_id", ""),
+                parent_id=props.get("parent_id"),
+                execution_id=props.get("execution_id", self.execution_id),
+                session_id=props.get("session_id"),
+                project_id=props.get("project_id"),
+                trace_id=props.get("trace_id", ""),
+                node_type=props.get("node_type", "Unknown"),
+                name=props.get("name", "unnamed"),
+                model=props.get("model"),
+                tool_name=props.get("tool_name"),
+                tool_desc=props.get("tool_desc"),
+                retrieved_docs=props.get("retrieved_docs"),
+                available_tools=props.get("available_tools"),
+                tool_calls=props.get("tool_calls"),
+                input=props.get("input"),
+                output=props.get("output"),
+                prompt=props.get("prompt"),
+                response=props.get("response"),
+                latency_ms=props.get("latency_ms", 0.0),
+                tokens=tokens,
+                error=props.get("error"),
+                status=props.get("status", "UNSET"),
+                timestamp=props.get("timestamp"),
+                end_timestamp=props.get("end_timestamp"),
+                metadata=_safe_json_loads(props.get("metadata") or "{}"),
+            )
+            nodes.append(node)
+
+        edges = []
+        for re_ in raw_edges:
+            try:
+                edges.append(ExecutionEdge(
+                    source_id=re_["src"],
+                    target_id=re_["tgt"],
+                    edge_type=re_.get("rel", "CALLS"),
+                ))
+            except Exception:
+                pass
+
+        session_id = nodes[0].session_id if nodes else None
+        graph = ExecutionGraph(
+            execution_id=self.execution_id,
+            session_id=session_id,
+            project_id=self.project_id or (nodes[0].project_id if nodes else None),
+            nodes=nodes,
+            edges=edges,
+        )
+        return graph
+
+    def _persist_to_mongo(
+        self,
+        node: ExecutionNode,
+        summary: NodeEvaluationSummary,
+        results: list[EvaluationResult],
+    ) -> None:
+        """Write evaluation results to MongoDB."""
+        try:
+            # Write each EvaluationResult as a separate `evaluations` document
+            for result in results:
+                if result.status == "SKIP":
+                    continue
+                self._store.write_evaluation_result(result, project_id=node.project_id)
+
+            # Write flat boolean props + overall status to the node document
+            flat_props = summary.flat_eval_props()
+            all_results_json = [r.to_dict() for r in results]
+            self._store.update_node_evaluation(
+                node_id=node.node_id,
+                flat_props=flat_props,
+                all_results=all_results_json,
+            )
+        except Exception as exc:
+            print(f"[EvaluationRunner] ⚠ MongoDB write failed for {node.name}: {exc}")
+
+    def _write_jsonl(self, results: list[EvaluationResult]) -> None:
+        """Append all evaluation results to JSONL file."""
+        self.output_path.parent.mkdir(parents=True, exist_ok=True)
+        with self.output_path.open("a", encoding="utf-8") as f:
+            for result in results:
+                f.write(json.dumps(result.to_dict(), default=str) + "\n")
+
+    def _run_rca(self, graph: ExecutionGraph, summaries: list[NodeEvaluationSummary], router: LLMRouter) -> dict:
+        """Trigger the RCA Engine and return its report as a dict."""
+        try:
+            from rca_engine.rca_engine import RCAEngine
+            # Attach evaluation results to graph nodes so RCA can use them
+            for summary in summaries:
+                node = graph.get_node(summary.node_id)
+                if node:
+                    node.validation_results = [r.to_dict() for r in summary.all_results()]
+
+            engine = RCAEngine(router)
+            rca_result = engine.analyze(graph)
+            return rca_result.model_dump()
+        except Exception as exc:
+            print(f"[EvaluationRunner] ⚠ RCA Engine failed: {exc}")
+            return {"error": str(exc), "overall_status": "UNKNOWN"}
+
+
+# ── Pretty printer ──────────────────────────────────────────────────────────
+
+_STATUS_ICONS = {"PASS": "✅", "FAIL": "❌", "WARNING": "⚠️ ", "SKIP": "⏭️ "}
+
+
+def _print_result(label: str, result: EvaluationResult) -> None:
+    icon = _STATUS_ICONS.get(result.status, "?")
+    sev = f" [{result.severity}]" if result.severity else ""
+    print(f"    {label}: {icon} {result.status}{sev}  (conf={result.confidence:.2f})")
+    if result.status not in ("PASS", "SKIP"):
+        print(f"            → {result.reason[:120]}")

evaluators/tool_agent_evaluator.py

@@ -0,0 +1,305 @@
+"""
+tool_agent_evaluator.py
+-----------------------
+Evaluator 3 — Tool / Agent Evaluator
+
+Validates agent nodes that call tools or sub-agents.
+Runs on: Agent nodes AND Tool/Retriever nodes.
+
+For AGENT nodes (that have tool/agent children):
+  Evaluates tool selection — was the right tool chosen?
+
+For TOOL / RETRIEVER nodes:
+  Walks up to find the parent agent to understand why the tool was called,
+  then validates:
+  - Input passed to the tool: was it correct and complete?
+  - Output returned from the tool: was it relevant and usable?
+
+checks schema:
+{
+  "tool_selection": {
+    "verdict":       "PASS" | "FAIL" | "WARNING" | "N/A",
+    "is_right_tool": bool,
+    "is_necessary":  bool,
+    "alternatives":  [str],
+    "reason":        str
+  },
+  "input_to_tool": {
+    "verdict":    "PASS" | "FAIL" | "WARNING" | "N/A",
+    "is_correct": bool,
+    "is_complete": bool,
+    "issues":     [str]
+  },
+  "output_from_tool": {
+    "verdict":     "PASS" | "FAIL" | "WARNING" | "N/A",
+    "is_relevant": bool,
+    "is_correct":  bool,
+    "issues":      [str]
+  }
+}
+"""
+
+from __future__ import annotations
+
+from graph_builder.models import ExecutionGraph, ExecutionNode, NodeType
+from .base_evaluator import BaseEvaluator
+from .models import EvaluationResult, EvaluationStatus, Severity
+
+
+_PROMPT_AGENT = """\
+You are an AI agent execution auditor. Evaluate an AGENT node that called one or more tools or sub-agents.
+
+## Agent Node
+{agent_context}
+
+## Tools / Sub-Agents Called
+{tools_called}
+
+## Instructions
+Evaluate whether the agent made GOOD TOOL SELECTION decisions.
+
+For each tool that was called, determine:
+1. **Is it the right tool?** — Given the task, was this the most appropriate tool to call?
+2. **Was it necessary?** — Was calling this tool required, or could the agent have answered without it?
+3. **Were there better alternatives?** — What other tools/agents could have been selected instead?
+
+## Severity Guide
+- CRITICAL: Tool selection will definitely cause wrong/harmful outcomes
+- HIGH: Clearly wrong tool chosen when a better one was available
+- MEDIUM: Tool works but a better choice existed
+- LOW: Tool choice is acceptable but slightly suboptimal
+
+## Response Format
+Respond ONLY with a valid JSON object. No explanation outside the JSON.
+
+{{
+  "status": "PASS" | "FAIL" | "WARNING",
+  "severity": "LOW" | "MEDIUM" | "HIGH" | "CRITICAL" | null,
+  "confidence": <float 0.0–1.0>,
+  "reason": "<1-2 sentence explanation>",
+  "suggestion": "<specific fix or null if PASS>",
+  "checks": {{
+    "tool_selection": {{
+      "verdict": "PASS" | "FAIL" | "WARNING",
+      "is_right_tool": true | false,
+      "is_necessary": true | false,
+      "alternatives": ["<better_tool_1>"],
+      "reason": "<explanation>"
+    }},
+    "input_to_tool": {{ "verdict": "N/A", "is_correct": null, "is_complete": null, "issues": [] }},
+    "output_from_tool": {{ "verdict": "N/A", "is_relevant": null, "is_correct": null, "issues": [] }}
+  }}
+}}
+"""
+
+_PROMPT_TOOL = """\
+You are an AI agent execution auditor. Evaluate a TOOL/RETRIEVER node execution.
+
+## Parent Agent Context (why this tool was called)
+{agent_context}
+
+## Tool Node Being Evaluated
+Name:        {tool_name}
+Description: {tool_desc}
+Input sent to tool:  {tool_input}
+Output from tool:    {tool_output}
+Status:      {tool_status}
+Error:       {tool_error}
+
+## Instructions
+Evaluate THREE things:
+
+1. **Input Quality** — Was the data sent TO this tool correct and complete?
+   - Were required parameters present and well-formed?
+   - Was the data accurate and relevant to the task?
+
+2. **Output Quality** — Was the result FROM this tool relevant and correct?
+   - Did the tool return useful data for the task?
+   - Is the output complete, or is it truncated/empty/errored?
+
+3. **Tool Selection** — Given the agent's goal, was this the right tool to call?
+   (You already have the parent agent context to judge this)
+
+## Severity Guide
+- CRITICAL: Tool returned harmful data, or catastrophic input caused data corruption/security issue
+- HIGH: Input was completely wrong causing tool failure, or output is entirely irrelevant
+- MEDIUM: Partial input issues or partially relevant output
+- LOW: Minor quality issues
+
+## Response Format
+Respond ONLY with a valid JSON object. No explanation outside the JSON.
+
+{{
+  "status": "PASS" | "FAIL" | "WARNING",
+  "severity": "LOW" | "MEDIUM" | "HIGH" | "CRITICAL" | null,
+  "confidence": <float 0.0–1.0>,
+  "reason": "<1-2 sentence explanation>",
+  "suggestion": "<specific fix or null if PASS>",
+  "checks": {{
+    "tool_selection": {{
+      "verdict": "PASS" | "FAIL" | "WARNING",
+      "is_right_tool": true | false,
+      "is_necessary": true | false,
+      "alternatives": [],
+      "reason": "<explanation>"
+    }},
+    "input_to_tool": {{
+      "verdict": "PASS" | "FAIL" | "WARNING",
+      "is_correct": true | false,
+      "is_complete": true | false,
+      "issues": ["<issue1>"]
+    }},
+    "output_from_tool": {{
+      "verdict": "PASS" | "FAIL" | "WARNING",
+      "is_relevant": true | false,
+      "is_correct": true | false,
+      "issues": ["<issue1>"]
+    }}
+  }}
+}}
+"""
+
+
+_TOOL_TYPES = {NodeType.Tool.value, NodeType.Retriever.value, "Tool", "Retriever"}
+_AGENT_TYPES = {NodeType.Agent.value, NodeType.Router.value, "Agent", "Router"}
+
+
+class ToolAgentEvaluator(BaseEvaluator):
+    """Evaluator 3: Validates tool/sub-agent selection, input quality, and output quality."""
+
+    name = "ToolAgentEvaluator"
+
+    def should_run(self, node: ExecutionNode, graph: ExecutionGraph) -> bool:
+        # Run on Tool/Retriever nodes
+        if node.node_type in _TOOL_TYPES:
+            return True
+        # Run on any node that itself decided to call a tool — node-local,
+        # sourced straight off the LLM span's own tool_calls attribute, so
+        # this works regardless of framework/node_type (raw OpenAI/Anthropic
+        # SDK agents never emit a separate Tool-kind child span, so they'd
+        # otherwise never be evaluated at all).
+        if node.tool_calls:
+            return True
+        # Fallback: Agent/Router nodes with a Tool/Retriever child in the
+        # graph — covers frameworks (e.g. LangChain's AgentExecutor) whose
+        # wrapper chain node doesn't itself carry the raw tool_calls
+        # attribute, and doesn't expose the individual reasoning iterations
+        # as their own LLM spans either.
+        if node.node_type in _AGENT_TYPES:
+            children = self._get_children(node, graph)
+            has_tool_child = any(c.node_type in _TOOL_TYPES for c in children)
+            if not has_tool_child:
+                return False
+            # Some frameworks (CrewAI) report every reasoning/tool-call
+            # iteration of a multi-step task as its own LLM-kind sibling
+            # span under this same Agent/Task node, each already evaluated
+            # individually via the node-local tool_calls check above. In
+            # that case, this coarse whole-node fallback would re-bundle
+            # tool calls from separate, unrelated decision points into one
+            # out-of-context verdict — skip it; the granular evaluations
+            # already cover it correctly.
+            iterations_evaluated_individually = any(
+                c.node_type == NodeType.LLM.value and c.tool_calls for c in children
+            )
+            return not iterations_evaluated_individually
+        return False
+
+    def _run(self, node: ExecutionNode, graph: ExecutionGraph) -> EvaluationResult:
+        if node.node_type in _TOOL_TYPES:
+            return self._evaluate_tool_node(node, graph)
+        else:
+            return self._evaluate_agent_node(node, graph)
+
+    def _evaluate_tool_node(
+        self, node: ExecutionNode, graph: ExecutionGraph
+    ) -> EvaluationResult:
+        """Evaluate from the perspective of the tool that was called."""
+        parent = self._get_parent(node, graph)
+        agent_context = (
+            self._format_node(parent)
+            if parent
+            else "(No parent agent context available)"
+        )
+
+        prompt = _PROMPT_TOOL.format(
+            agent_context=agent_context,
+            tool_name=node.tool_name or node.name,
+            tool_desc=node.tool_desc or "(No description)",
+            tool_input=self._safe_json(node.input)[:2000],
+            tool_output=self._safe_json(node.output)[:2000],
+            tool_status=node.status,
+            tool_error=node.error or "(none)",
+        )
+
+        return self._parse_and_build(prompt, node)
+
+    def _evaluate_agent_node(
+        self, node: ExecutionNode, graph: ExecutionGraph
+    ) -> EvaluationResult:
+        """Evaluate from the perspective of an agent/LLM node that called tools."""
+        tools_called_lines = []
+
+        # Primary source: the tool-call decision(s) captured directly on this
+        # node's own LLM span attributes — framework-agnostic, requires no
+        # child Tool span to exist.
+        for call in node.tool_calls or []:
+            tools_called_lines.append(
+                f"  Tool: {call.get('name')}\n"
+                f"  Arguments: {self._safe_json(call.get('arguments'))[:500]}\n"
+            )
+
+        # Secondary enrichment: if a real Tool/Retriever execution span does
+        # exist as a child (e.g. LangChain), include its actual input/output
+        # for richer context — optional, not required to run this evaluator.
+        children = self._get_children(node, graph)
+        tool_children = [c for c in children if c.node_type in _TOOL_TYPES]
+        for tc in tool_children:
+            tools_called_lines.append(
+                f"  Tool: {tc.tool_name or tc.name}\n"
+                f"  Input: {self._safe_json(tc.input)[:500]}\n"
+                f"  Output: {self._safe_json(tc.output)[:500]}\n"
+                f"  Status: {tc.status}\n"
+            )
+
+        tools_called = "\n".join(tools_called_lines) if tools_called_lines else "(none)"
+
+        agent_context = self._format_node(node)
+        if node.available_tools:
+            available_desc = ", ".join(
+                (t.get("function", {}).get("name") if isinstance(t, dict) else str(t))
+                for t in node.available_tools
+            )
+            agent_context += f"\nTools available to choose from: {available_desc}"
+
+        prompt = _PROMPT_AGENT.format(
+            agent_context=agent_context,
+            tools_called=tools_called,
+        )
+
+        return self._parse_and_build(prompt, node)
+
+    def _parse_and_build(self, prompt: str, node: ExecutionNode) -> EvaluationResult:
+        parsed = self._call_and_parse(prompt)
+
+        status_raw = parsed.get("status", "FAIL")
+        try:
+            status = EvaluationStatus(status_raw)
+        except ValueError:
+            status = EvaluationStatus.FAIL
+
+        severity = self._severity_from_status(parsed, default=Severity.MEDIUM)
+
+        return EvaluationResult(
+            evaluator=self.name,
+            node_id=node.node_id,
+            node_name=node.name,
+            node_type=node.node_type,
+            execution_id=node.execution_id,
+            session_id=node.session_id,
+            status=status,
+            severity=severity if status != EvaluationStatus.PASS else None,
+            confidence=float(parsed.get("confidence", 0.5)),
+            reason=parsed.get("reason", ""),
+            suggestion=parsed.get("suggestion"),
+            checks=parsed.get("checks", {}),
+        )

graph_builder/__init__.py

@@ -0,0 +1,6 @@
+"""Graph builder package — converts execution traces into a MongoDB-backed graph."""
+from .models import ExecutionNode, ExecutionEdge, ExecutionGraph
+from .builder import TraceToGraphBuilder
+from .mongo_store import MongoStore
+
+__all__ = ["ExecutionNode", "ExecutionEdge", "ExecutionGraph", "TraceToGraphBuilder", "MongoStore"]