cortexops 0.1.0__py3-none-any.whl

cortexops/eval.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import statistics
+import time
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .metrics import compute_case_result
+from .models import (
+    EvalCase,
+    EvalDataset,
+    EvalSummary,
+    FailureKind,
+    RunStatus,
+    Trace,
+    TraceNode,
+)
+
+
+class EvalSuite:
+    """Run evaluation suites against any instrumented agent.
+
+    Usage:
+        results = EvalSuite.run(
+            dataset="golden_v1.yaml",
+            agent=your_langgraph_app,   # wrapped or raw callable
+        )
+        print(results.summary())
+    """
+
+    @classmethod
+    def run(
+        cls,
+        dataset: str | Path | dict | EvalDataset,
+        agent: Any,
+        *,
+        metrics: list[str] | None = None,
+        verbose: bool = True,
+        fail_on: str | None = None,
+    ) -> EvalSummary:
+        """Run a full eval suite.
+
+        Args:
+            dataset:  Path to YAML, dict, or EvalDataset object.
+            agent:    Any callable that accepts a string or dict input.
+            metrics:  Optional subset of metrics to run.
+            verbose:  Print case-by-case progress.
+            fail_on:  Threshold expression like "task_completion < 0.90".
+                      Raises EvalThresholdError if the condition is met.
+        """
+        ds = cls._load_dataset(dataset)
+        case_results = []
+
+        for i, case in enumerate(ds.cases):
+            if verbose:
+                print(f"  [{i+1}/{len(ds.cases)}] {case.id} ... ", end="", flush=True)
+
+            trace = cls._run_case(agent, case)
+            result = compute_case_result(case, trace)
+            case_results.append(result)
+
+            if verbose:
+                status = "pass" if result.passed else "FAIL"
+                print(f"{status} ({result.score:.0f})")
+
+        latencies = [r.latency_ms for r in case_results]
+        latencies_sorted = sorted(latencies)
+        n = len(latencies_sorted)
+
+        summary = EvalSummary(
+            project=ds.project,
+            dataset_version=ds.version,
+            total_cases=len(case_results),
+            passed=sum(1 for r in case_results if r.passed),
+            failed=sum(1 for r in case_results if not r.passed),
+            warnings=sum(1 for r in case_results if not r.passed and r.score >= 60),
+            task_completion_rate=sum(1 for r in case_results if r.task_completion) / max(n, 1),
+            tool_accuracy=statistics.mean(r.tool_accuracy for r in case_results) if case_results else 0.0,
+            latency_p50_ms=latencies_sorted[int(n * 0.50) - 1] if n else 0.0,
+            latency_p95_ms=latencies_sorted[int(n * 0.95) - 1] if n else 0.0,
+            case_results=case_results,
+        )
+
+        if verbose:
+            print()
+            print(summary.summary())
+
+        if fail_on:
+            cls._check_threshold(summary, fail_on)
+
+        return summary
+
+    @classmethod
+    def _run_case(cls, agent: Any, case: EvalCase) -> Trace:
+        input_data = case.input if isinstance(case.input, dict) else {"input": case.input}
+        t0 = time.perf_counter()
+
+        try:
+            if hasattr(agent, "invoke"):
+                output = agent.invoke(input_data)
+            elif callable(agent):
+                output = agent(input_data)
+            else:
+                raise TypeError(f"Agent {type(agent).__name__} is not callable")
+            latency_ms = (time.perf_counter() - t0) * 1000
+            output_dict = output if isinstance(output, dict) else {"output": str(output)}
+
+            return Trace(
+                project="eval",
+                case_id=case.id,
+                input=input_data,
+                output=output_dict,
+                total_latency_ms=latency_ms,
+                status=RunStatus.COMPLETED,
+                nodes=[
+                    TraceNode(
+                        node_id="eval_root",
+                        node_name="agent",
+                        input=input_data,
+                        output=output_dict,
+                        latency_ms=latency_ms,
+                    )
+                ],
+            )
+
+        except Exception as exc:
+            latency_ms = (time.perf_counter() - t0) * 1000
+            return Trace(
+                project="eval",
+                case_id=case.id,
+                input=input_data,
+                output={},
+                total_latency_ms=latency_ms,
+                status=RunStatus.FAILED,
+                failure_kind=FailureKind.UNKNOWN,
+                failure_detail=str(exc),
+            )
+
+    @classmethod
+    def _load_dataset(cls, dataset: str | Path | dict | EvalDataset) -> EvalDataset:
+        if isinstance(dataset, EvalDataset):
+            return dataset
+
+        if isinstance(dataset, dict):
+            return cls._parse_dataset_dict(dataset)
+
+        path = Path(dataset)
+        if not path.exists():
+            raise FileNotFoundError(f"Dataset not found: {path}")
+
+        raw = yaml.safe_load(path.read_text())
+        return cls._parse_dataset_dict(raw)
+
+    @classmethod
+    def _parse_dataset_dict(cls, raw: dict) -> EvalDataset:
+        cases = []
+        for c in raw.get("cases", []):
+            cases.append(
+                EvalCase(
+                    id=str(c["id"]),
+                    input=c["input"],
+                    expected_tool_calls=c.get("expected_tool_calls", []),
+                    expected_output_contains=c.get("expected_output_contains", []),
+                    expected_output_not_contains=c.get("expected_output_not_contains", []),
+                    max_latency_ms=c.get("max_latency_ms"),
+                    judge=c.get("judge", "rule"),
+                    judge_criteria=c.get("judge_criteria"),
+                    tags=c.get("tags", []),
+                )
+            )
+        return EvalDataset(
+            version=raw.get("version", 1),
+            project=raw.get("project", "unknown"),
+            description=raw.get("description", ""),
+            cases=cases,
+        )
+
+    @classmethod
+    def _check_threshold(cls, summary: EvalSummary, fail_on: str) -> None:
+        """Parse and evaluate a threshold expression like 'task_completion < 0.90'."""
+        import re
+
+        m = re.match(r"(\w+)\s*([<>]=?)\s*([\d.]+)", fail_on.strip())
+        if not m:
+            raise ValueError(f"Invalid fail_on expression: '{fail_on}'")
+
+        metric, op, threshold_str = m.groups()
+        threshold = float(threshold_str)
+
+        actual = {
+            "task_completion": summary.task_completion_rate,
+            "tool_accuracy": summary.tool_accuracy / 100.0,
+            "pass_rate": summary.passed / max(summary.total_cases, 1),
+        }.get(metric)
+
+        if actual is None:
+            raise ValueError(f"Unknown metric in fail_on: '{metric}'")
+
+        failed = {
+            "<": actual < threshold,
+            "<=": actual <= threshold,
+            ">": actual > threshold,
+            ">=": actual >= threshold,
+        }.get(op, False)
+
+        if failed:
+            raise EvalThresholdError(
+                f"Eval gate failed: {metric}={actual:.3f} {op} {threshold} "
+                f"(project={summary.project})"
+            )
+
+
+class EvalThresholdError(Exception):
+    """Raised when an eval run fails a CI threshold gate."""

cortexops/judge.py

@@ -0,0 +1,155 @@
+"""LLM-as-judge metric for CortexOps.
+
+Uses an LLM (default: gpt-4o-mini) to score open-ended agent output
+against natural language criteria. Works with any OpenAI-compatible API.
+
+Usage in golden dataset:
+    - id: refund_explanation_01
+      input: "Why was my refund rejected?"
+      judge: llm
+      judge_criteria: >
+        The response should explain the rejection reason clearly,
+        be empathetic, and offer a next step to the customer.
+        It must NOT contain jargon or mention internal system errors.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from .metrics import Metric
+from .models import EvalCase, FailureKind, Trace
+
+
+JUDGE_SYSTEM_PROMPT = """You are a strict but fair evaluator of AI agent outputs.
+You will be given:
+  - The user's input to the agent
+  - The agent's output
+  - Evaluation criteria
+
+Score the output from 0 to 100 and explain your reasoning briefly.
+
+Respond ONLY with valid JSON in this exact format:
+{
+  "score": <integer 0-100>,
+  "passed": <true|false>,
+  "reasoning": "<one sentence>"
+}
+
+Rules:
+- 90-100: Fully meets all criteria, no issues
+- 70-89: Mostly meets criteria, minor gaps
+- 50-69: Partially meets criteria, notable gaps
+- 0-49: Fails to meet criteria or contains harmful/incorrect content
+- passed = true only if score >= 70
+"""
+
+
+class LLMJudgeMetric(Metric):
+    """Score agent output using an LLM judge.
+
+    Falls back to a heuristic score if the LLM API is unavailable,
+    so evals never block on API failures.
+
+    Args:
+        model:          OpenAI model to use. Default: gpt-4o-mini.
+        api_key:        OpenAI API key. Falls back to OPENAI_API_KEY env var.
+        base_url:       OpenAI-compatible base URL. Useful for local LLMs.
+        temperature:    Judge temperature. Keep low (0.1) for consistency.
+        timeout:        HTTP timeout in seconds.
+    """
+
+    name = "llm_judge"
+
+    def __init__(
+        self,
+        model: str = "gpt-4o-mini",
+        api_key: str | None = None,
+        base_url: str = "https://api.openai.com/v1",
+        temperature: float = 0.1,
+        timeout: float = 30.0,
+    ) -> None:
+        self.model = model
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY") or os.getenv("CORTEXOPS_JUDGE_API_KEY")
+        self.base_url = base_url.rstrip("/")
+        self.temperature = temperature
+        self.timeout = timeout
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if not case.judge_criteria:
+            return 100.0, None, None
+
+        if case.judge != "llm":
+            return 100.0, None, None
+
+        user_input = str(case.input)
+        agent_output = str(trace.output.get("output", trace.output))
+
+        try:
+            result = self._call_judge(user_input, agent_output, case.judge_criteria)
+            score = float(result.get("score", 0))
+            passed = result.get("passed", score >= 70)
+            reasoning = result.get("reasoning", "")
+
+            if not passed:
+                return score, FailureKind.OUTPUT_FORMAT, f"LLM judge: {reasoning}"
+            return score, None, None
+
+        except Exception as exc:
+            return self._heuristic_fallback(case, trace, str(exc))
+
+    def _call_judge(self, user_input: str, agent_output: str, criteria: str) -> dict[str, Any]:
+        import httpx
+
+        if not self.api_key:
+            raise ValueError(
+                "No API key found for LLM judge. Set OPENAI_API_KEY or pass api_key= to LLMJudgeMetric()."
+            )
+
+        user_message = (
+            f"USER INPUT:\n{user_input}\n\n"
+            f"AGENT OUTPUT:\n{agent_output}\n\n"
+            f"EVALUATION CRITERIA:\n{criteria}"
+        )
+
+        response = httpx.post(
+            f"{self.base_url}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            },
+            json={
+                "model": self.model,
+                "messages": [
+                    {"role": "system", "content": JUDGE_SYSTEM_PROMPT},
+                    {"role": "user", "content": user_message},
+                ],
+                "temperature": self.temperature,
+                "response_format": {"type": "json_object"},
+            },
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+        return json.loads(content)
+
+    def _heuristic_fallback(
+        self, case: EvalCase, trace: Trace, error: str
+    ) -> tuple[float, FailureKind | None, str | None]:
+        """Simple keyword fallback when the LLM is unavailable."""
+        output = str(trace.output.get("output", "")).lower()
+        criteria_words = (case.judge_criteria or "").lower().split()
+        meaningful_words = [w for w in criteria_words if len(w) > 4]
+
+        if not meaningful_words:
+            return 70.0, None, f"LLM judge unavailable ({error[:60]}); heuristic used"
+
+        hits = sum(1 for w in meaningful_words if w in output)
+        ratio = hits / len(meaningful_words)
+        score = 50.0 + 50.0 * ratio
+
+        if score < 70:
+            return score, FailureKind.OUTPUT_FORMAT, f"LLM judge unavailable; heuristic score {score:.0f}"
+        return score, None, f"LLM judge unavailable ({error[:60]}); heuristic used"

cortexops/metrics.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .models import CaseResult, EvalCase, FailureKind, Trace
+
+
+class Metric(ABC):
+    """Base class for all CortexOps eval metrics.
+    Subclass this and implement score() to add custom metrics.
+    """
+
+    name: str = "base"
+
+    @abstractmethod
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        """Return (score 0-100, failure_kind or None, failure_detail or None)."""
+
+
+class TaskCompletionMetric(Metric):
+    """Checks whether the agent produced a non-empty, non-error output."""
+
+    name = "task_completion"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        output = trace.output or {}
+        output_str = str(output.get("output", output.get("result", output.get("answer", ""))))
+
+        if not output_str or output_str.lower() in ("none", "null", ""):
+            return 0.0, FailureKind.UNKNOWN, "Agent produced no output"
+
+        error_patterns = [r"error:", r"exception:", r"traceback", r"failed to"]
+        for pat in error_patterns:
+            if re.search(pat, output_str, re.IGNORECASE):
+                return 20.0, FailureKind.UNKNOWN, f"Output contains error signal: {output_str[:100]}"
+
+        if case.expected_output_contains:
+            hits = sum(1 for kw in case.expected_output_contains if kw.lower() in output_str.lower())
+            ratio = hits / len(case.expected_output_contains)
+            if ratio < 1.0:
+                missing = [kw for kw in case.expected_output_contains if kw.lower() not in output_str.lower()]
+                return (
+                    50.0 + 50.0 * ratio,
+                    FailureKind.OUTPUT_FORMAT,
+                    f"Missing expected content: {missing}",
+                )
+
+        if case.expected_output_not_contains:
+            violations = [kw for kw in case.expected_output_not_contains if kw.lower() in output_str.lower()]
+            if violations:
+                return (
+                    30.0,
+                    FailureKind.HALLUCINATION,
+                    f"Output contains prohibited content: {violations}",
+                )
+
+        return 100.0, None, None
+
+
+class ToolAccuracyMetric(Metric):
+    """Checks whether expected tool calls were actually made.
+
+    Looks in two places (in priority order):
+    1. Trace node tool_calls (full instrumentation via CortexTracer.record_tool_call)
+    2. output['tool_calls_made'] list (lightweight self-reporting from the agent)
+    """
+
+    name = "tool_accuracy"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if not case.expected_tool_calls:
+            return 100.0, None, None
+
+        # Priority 1: instrumented trace nodes
+        actual_calls = {tc.name for tc in trace.tool_calls()}
+
+        # Priority 2: agent self-reported via output dict
+        if not actual_calls:
+            reported = trace.output.get("tool_calls_made", [])
+            if isinstance(reported, list):
+                actual_calls = set(reported)
+
+        expected = set(case.expected_tool_calls)
+        missing = expected - actual_calls
+
+        if not missing:
+            return 100.0, None, None
+
+        ratio = len(expected - missing) / len(expected)
+        return (
+            round(ratio * 100, 1),
+            FailureKind.TOOL_CALL_MISMATCH,
+            f"Missing tool calls: {sorted(missing)}",
+        )
+
+
+class LatencyMetric(Metric):
+    """Checks whether the agent responded within the required latency budget."""
+
+    name = "latency"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if case.max_latency_ms is None:
+            return 100.0, None, None
+        if trace.total_latency_ms <= case.max_latency_ms:
+            return 100.0, None, None
+        overage = trace.total_latency_ms - case.max_latency_ms
+        return (
+            max(0.0, 100.0 - (overage / case.max_latency_ms) * 100),
+            FailureKind.TIMEOUT,
+            f"Latency {trace.total_latency_ms:.0f}ms exceeded budget {case.max_latency_ms:.0f}ms",
+        )
+
+
+class HallucinationMetric(Metric):
+    """Detects common hallucination signals in agent output.
+    Flags confident fabrications, contradictions, and forbidden facts.
+    """
+
+    name = "hallucination"
+
+    HALLUCINATION_PATTERNS = [
+        r"\bas of (january|february|march|april|may|june|july|august|september|october|november|december) 20[0-9]{2}\b",
+        r"\bi (don't|do not) have (access|information|data)\b",
+        r"\bi cannot (access|retrieve|look up)\b",
+    ]
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        output = str(trace.output)
+        for pat in self.HALLUCINATION_PATTERNS:
+            if re.search(pat, output, re.IGNORECASE):
+                return (
+                    40.0,
+                    FailureKind.HALLUCINATION,
+                    f"Hallucination signal detected: pattern '{pat}'",
+                )
+        return 100.0, None, None
+
+
+def compute_case_result(case: EvalCase, trace: Trace, extra_metrics: "list[Metric] | None" = None) -> CaseResult:
+    metrics: list[Metric] = [
+        TaskCompletionMetric(),
+        ToolAccuracyMetric(),
+        LatencyMetric(),
+        HallucinationMetric(),
+    ]
+
+    if case.judge == "llm":
+        from .judge import LLMJudgeMetric
+        metrics.append(LLMJudgeMetric())
+
+    if extra_metrics:
+        metrics.extend(extra_metrics)
+
+    scores: list[float] = []
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+
+    for metric in metrics:
+        s, fk, fd = metric.score(case, trace)
+        scores.append(s)
+        if s < 100.0 and failure_kind is None:
+            failure_kind = fk
+            failure_detail = fd
+
+    final_score = sum(scores) / len(scores)
+    task_ok_score, _, _ = TaskCompletionMetric().score(case, trace)
+    tool_score, _, _ = ToolAccuracyMetric().score(case, trace)
+    lat_score, _, _ = LatencyMetric().score(case, trace)
+
+    return CaseResult(
+        case_id=case.id,
+        passed=final_score >= 80.0,
+        score=round(final_score, 1),
+        task_completion=task_ok_score >= 80.0,
+        tool_accuracy=round(tool_score, 1),
+        latency_ms=trace.total_latency_ms,
+        latency_ok=lat_score >= 80.0,
+        failure_kind=failure_kind,
+        failure_detail=failure_detail,
+        trace=trace,
+    )

cortexops/models.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class RunStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class ToolCallStatus(str, Enum):
+    SUCCESS = "success"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+
+
+class FailureKind(str, Enum):
+    TOOL_CALL_MISMATCH = "tool_call_mismatch"
+    HALLUCINATION = "hallucination"
+    PLAN_DEVIATION = "plan_deviation"
+    TIMEOUT = "timeout"
+    CONTEXT_OVERFLOW = "context_overflow"
+    OUTPUT_FORMAT = "output_format"
+    UNKNOWN = "unknown"
+
+
+class ToolCall(BaseModel):
+    name: str
+    args: dict[str, Any] = Field(default_factory=dict)
+    result: Any = None
+    status: ToolCallStatus = ToolCallStatus.SUCCESS
+    latency_ms: float = 0.0
+    error: str | None = None
+
+
+class TraceNode(BaseModel):
+    node_id: str
+    node_name: str
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: dict[str, Any] = Field(default_factory=dict)
+    tool_calls: list[ToolCall] = Field(default_factory=list)
+    llm_prompt: str | None = None
+    llm_response: str | None = None
+    latency_ms: float = 0.0
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Trace(BaseModel):
+    trace_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    project: str
+    run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    case_id: str | None = None
+    nodes: list[TraceNode] = Field(default_factory=list)
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: dict[str, Any] = Field(default_factory=dict)
+    total_latency_ms: float = 0.0
+    status: RunStatus = RunStatus.COMPLETED
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+    def tool_calls(self) -> list[ToolCall]:
+        return [tc for node in self.nodes for tc in node.tool_calls]
+
+    def total_tool_calls(self) -> int:
+        return len(self.tool_calls())
+
+
+class EvalCase(BaseModel):
+    id: str
+    input: str | dict[str, Any]
+    expected_tool_calls: list[str] = Field(default_factory=list)
+    expected_output_contains: list[str] = Field(default_factory=list)
+    expected_output_not_contains: list[str] = Field(default_factory=list)
+    max_latency_ms: float | None = None
+    judge: str = "rule"  # "rule" | "llm"
+    judge_criteria: str | None = None
+    tags: list[str] = Field(default_factory=list)
+
+
+class EvalDataset(BaseModel):
+    version: int = 1
+    project: str
+    description: str = ""
+    cases: list[EvalCase] = Field(default_factory=list)
+
+
+class CaseResult(BaseModel):
+    case_id: str
+    passed: bool
+    score: float  # 0.0 - 100.0
+    task_completion: bool
+    tool_accuracy: float
+    latency_ms: float
+    latency_ok: bool
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+    trace: Trace | None = None
+
+
+class EvalSummary(BaseModel):
+    run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    project: str
+    dataset_version: int
+    total_cases: int
+    passed: int
+    failed: int
+    warnings: int
+    task_completion_rate: float
+    tool_accuracy: float
+    latency_p50_ms: float
+    latency_p95_ms: float
+    regressions: int = 0
+    baseline_run_id: str | None = None
+    case_results: list[CaseResult] = Field(default_factory=list)
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+    def summary(self) -> str:
+        lines = [
+            f"CortexOps eval — {self.project}",
+            f"  Run ID          : {self.run_id}",
+            f"  Cases           : {self.total_cases}  ({self.passed} passed, {self.failed} failed)",
+            f"  Task completion : {self.task_completion_rate:.1%}",
+            f"  Tool accuracy   : {self.tool_accuracy:.1f}/100",
+            f"  Latency p50/p95 : {self.latency_p50_ms:.0f}ms / {self.latency_p95_ms:.0f}ms",
+        ]
+        if self.regressions:
+            lines.append(f"  Regressions     : {self.regressions}  (vs baseline {self.baseline_run_id})")
+        failing = [r for r in self.case_results if not r.passed]
+        if failing:
+            lines.append("  Failed cases:")
+            for r in failing:
+                lines.append(f"    - {r.case_id}: {r.failure_kind or 'unknown'} (score {r.score:.0f})")
+        return "\n".join(lines)