cortexops 0.1.0__py3-none-any.whl

cortexops/cortexops/metrics.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .models import CaseResult, EvalCase, FailureKind, Trace
+
+
+class Metric(ABC):
+    """Base class for all CortexOps eval metrics.
+    Subclass this and implement score() to add custom metrics.
+    """
+
+    name: str = "base"
+
+    @abstractmethod
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        """Return (score 0-100, failure_kind or None, failure_detail or None)."""
+
+
+class TaskCompletionMetric(Metric):
+    """Checks whether the agent produced a non-empty, non-error output."""
+
+    name = "task_completion"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        output = trace.output or {}
+        output_str = str(output.get("output", output.get("result", output.get("answer", ""))))
+
+        if not output_str or output_str.lower() in ("none", "null", ""):
+            return 0.0, FailureKind.UNKNOWN, "Agent produced no output"
+
+        error_patterns = [r"error:", r"exception:", r"traceback", r"failed to"]
+        for pat in error_patterns:
+            if re.search(pat, output_str, re.IGNORECASE):
+                return 20.0, FailureKind.UNKNOWN, f"Output contains error signal: {output_str[:100]}"
+
+        if case.expected_output_contains:
+            hits = sum(1 for kw in case.expected_output_contains if kw.lower() in output_str.lower())
+            ratio = hits / len(case.expected_output_contains)
+            if ratio < 1.0:
+                missing = [kw for kw in case.expected_output_contains if kw.lower() not in output_str.lower()]
+                return (
+                    50.0 + 50.0 * ratio,
+                    FailureKind.OUTPUT_FORMAT,
+                    f"Missing expected content: {missing}",
+                )
+
+        if case.expected_output_not_contains:
+            violations = [kw for kw in case.expected_output_not_contains if kw.lower() in output_str.lower()]
+            if violations:
+                return (
+                    30.0,
+                    FailureKind.HALLUCINATION,
+                    f"Output contains prohibited content: {violations}",
+                )
+
+        return 100.0, None, None
+
+
+class ToolAccuracyMetric(Metric):
+    """Checks whether expected tool calls were actually made.
+
+    Looks in two places (in priority order):
+    1. Trace node tool_calls (full instrumentation via CortexTracer.record_tool_call)
+    2. output['tool_calls_made'] list (lightweight self-reporting from the agent)
+    """
+
+    name = "tool_accuracy"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if not case.expected_tool_calls:
+            return 100.0, None, None
+
+        # Priority 1: instrumented trace nodes
+        actual_calls = {tc.name for tc in trace.tool_calls()}
+
+        # Priority 2: agent self-reported via output dict
+        if not actual_calls:
+            reported = trace.output.get("tool_calls_made", [])
+            if isinstance(reported, list):
+                actual_calls = set(reported)
+
+        expected = set(case.expected_tool_calls)
+        missing = expected - actual_calls
+
+        if not missing:
+            return 100.0, None, None
+
+        ratio = len(expected - missing) / len(expected)
+        return (
+            round(ratio * 100, 1),
+            FailureKind.TOOL_CALL_MISMATCH,
+            f"Missing tool calls: {sorted(missing)}",
+        )
+
+
+class LatencyMetric(Metric):
+    """Checks whether the agent responded within the required latency budget."""
+
+    name = "latency"
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if case.max_latency_ms is None:
+            return 100.0, None, None
+        if trace.total_latency_ms <= case.max_latency_ms:
+            return 100.0, None, None
+        overage = trace.total_latency_ms - case.max_latency_ms
+        return (
+            max(0.0, 100.0 - (overage / case.max_latency_ms) * 100),
+            FailureKind.TIMEOUT,
+            f"Latency {trace.total_latency_ms:.0f}ms exceeded budget {case.max_latency_ms:.0f}ms",
+        )
+
+
+class HallucinationMetric(Metric):
+    """Detects common hallucination signals in agent output.
+    Flags confident fabrications, contradictions, and forbidden facts.
+    """
+
+    name = "hallucination"
+
+    HALLUCINATION_PATTERNS = [
+        r"\bas of (january|february|march|april|may|june|july|august|september|october|november|december) 20[0-9]{2}\b",
+        r"\bi (don't|do not) have (access|information|data)\b",
+        r"\bi cannot (access|retrieve|look up)\b",
+    ]
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        output = str(trace.output)
+        for pat in self.HALLUCINATION_PATTERNS:
+            if re.search(pat, output, re.IGNORECASE):
+                return (
+                    40.0,
+                    FailureKind.HALLUCINATION,
+                    f"Hallucination signal detected: pattern '{pat}'",
+                )
+        return 100.0, None, None
+
+
+def compute_case_result(case: EvalCase, trace: Trace, extra_metrics: "list[Metric] | None" = None) -> CaseResult:
+    metrics: list[Metric] = [
+        TaskCompletionMetric(),
+        ToolAccuracyMetric(),
+        LatencyMetric(),
+        HallucinationMetric(),
+    ]
+
+    if case.judge == "llm":
+        from .judge import LLMJudgeMetric
+        metrics.append(LLMJudgeMetric())
+
+    if extra_metrics:
+        metrics.extend(extra_metrics)
+
+    scores: list[float] = []
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+
+    for metric in metrics:
+        s, fk, fd = metric.score(case, trace)
+        scores.append(s)
+        if s < 100.0 and failure_kind is None:
+            failure_kind = fk
+            failure_detail = fd
+
+    final_score = sum(scores) / len(scores)
+    task_ok_score, _, _ = TaskCompletionMetric().score(case, trace)
+    tool_score, _, _ = ToolAccuracyMetric().score(case, trace)
+    lat_score, _, _ = LatencyMetric().score(case, trace)
+
+    return CaseResult(
+        case_id=case.id,
+        passed=final_score >= 80.0,
+        score=round(final_score, 1),
+        task_completion=task_ok_score >= 80.0,
+        tool_accuracy=round(tool_score, 1),
+        latency_ms=trace.total_latency_ms,
+        latency_ok=lat_score >= 80.0,
+        failure_kind=failure_kind,
+        failure_detail=failure_detail,
+        trace=trace,
+    )

cortexops/cortexops/models.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class RunStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class ToolCallStatus(str, Enum):
+    SUCCESS = "success"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+
+
+class FailureKind(str, Enum):
+    TOOL_CALL_MISMATCH = "tool_call_mismatch"
+    HALLUCINATION = "hallucination"
+    PLAN_DEVIATION = "plan_deviation"
+    TIMEOUT = "timeout"
+    CONTEXT_OVERFLOW = "context_overflow"
+    OUTPUT_FORMAT = "output_format"
+    UNKNOWN = "unknown"
+
+
+class ToolCall(BaseModel):
+    name: str
+    args: dict[str, Any] = Field(default_factory=dict)
+    result: Any = None
+    status: ToolCallStatus = ToolCallStatus.SUCCESS
+    latency_ms: float = 0.0
+    error: str | None = None
+
+
+class TraceNode(BaseModel):
+    node_id: str
+    node_name: str
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: dict[str, Any] = Field(default_factory=dict)
+    tool_calls: list[ToolCall] = Field(default_factory=list)
+    llm_prompt: str | None = None
+    llm_response: str | None = None
+    latency_ms: float = 0.0
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Trace(BaseModel):
+    trace_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    project: str
+    run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    case_id: str | None = None
+    nodes: list[TraceNode] = Field(default_factory=list)
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: dict[str, Any] = Field(default_factory=dict)
+    total_latency_ms: float = 0.0
+    status: RunStatus = RunStatus.COMPLETED
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+    def tool_calls(self) -> list[ToolCall]:
+        return [tc for node in self.nodes for tc in node.tool_calls]
+
+    def total_tool_calls(self) -> int:
+        return len(self.tool_calls())
+
+
+class EvalCase(BaseModel):
+    id: str
+    input: str | dict[str, Any]
+    expected_tool_calls: list[str] = Field(default_factory=list)
+    expected_output_contains: list[str] = Field(default_factory=list)
+    expected_output_not_contains: list[str] = Field(default_factory=list)
+    max_latency_ms: float | None = None
+    judge: str = "rule"  # "rule" | "llm"
+    judge_criteria: str | None = None
+    tags: list[str] = Field(default_factory=list)
+
+
+class EvalDataset(BaseModel):
+    version: int = 1
+    project: str
+    description: str = ""
+    cases: list[EvalCase] = Field(default_factory=list)
+
+
+class CaseResult(BaseModel):
+    case_id: str
+    passed: bool
+    score: float  # 0.0 - 100.0
+    task_completion: bool
+    tool_accuracy: float
+    latency_ms: float
+    latency_ok: bool
+    failure_kind: FailureKind | None = None
+    failure_detail: str | None = None
+    trace: Trace | None = None
+
+
+class EvalSummary(BaseModel):
+    run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    project: str
+    dataset_version: int
+    total_cases: int
+    passed: int
+    failed: int
+    warnings: int
+    task_completion_rate: float
+    tool_accuracy: float
+    latency_p50_ms: float
+    latency_p95_ms: float
+    regressions: int = 0
+    baseline_run_id: str | None = None
+    case_results: list[CaseResult] = Field(default_factory=list)
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+    def summary(self) -> str:
+        lines = [
+            f"CortexOps eval — {self.project}",
+            f"  Run ID          : {self.run_id}",
+            f"  Cases           : {self.total_cases}  ({self.passed} passed, {self.failed} failed)",
+            f"  Task completion : {self.task_completion_rate:.1%}",
+            f"  Tool accuracy   : {self.tool_accuracy:.1f}/100",
+            f"  Latency p50/p95 : {self.latency_p50_ms:.0f}ms / {self.latency_p95_ms:.0f}ms",
+        ]
+        if self.regressions:
+            lines.append(f"  Regressions     : {self.regressions}  (vs baseline {self.baseline_run_id})")
+        failing = [r for r in self.case_results if not r.passed]
+        if failing:
+            lines.append("  Failed cases:")
+            for r in failing:
+                lines.append(f"    - {r.case_id}: {r.failure_kind or 'unknown'} (score {r.score:.0f})")
+        return "\n".join(lines)

cortexops/cortexops/tracer.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import time
+import uuid
+from contextlib import contextmanager
+from typing import Any, Callable
+
+from .models import FailureKind, RunStatus, Trace, TraceNode, ToolCall, ToolCallStatus
+
+
+class CortexTracer:
+    """Instruments AI agents with zero-refactor tracing.
+
+    Supports LangGraph StateGraph and CrewAI Crew out of the box.
+    Falls back to a generic callable wrapper for any other agent type.
+
+    Usage:
+        tracer = CortexTracer(project="payments-agent")
+        graph = tracer.wrap(your_langgraph_app)
+        result = graph.invoke({"messages": [...]})
+        trace  = tracer.last_trace()
+    """
+
+    def __init__(
+        self,
+        project: str,
+        api_key: str | None = None,
+        environment: str = "development",
+        sample_rate: float = 1.0,
+        local_store: bool = True,
+    ) -> None:
+        self.project = project
+        self.api_key = api_key
+        self.environment = environment
+        self.sample_rate = sample_rate
+        self.local_store = local_store
+        self._traces: list[Trace] = []
+        self._current_trace: Trace | None = None
+
+    def wrap(self, agent: Any) -> Any:
+        """Auto-detect agent type and return an instrumented wrapper."""
+        agent_type = type(agent).__name__
+
+        if agent_type == "CompiledStateGraph":
+            return self._wrap_langgraph(agent)
+
+        if agent_type == "Crew":
+            return self._wrap_crewai(agent)
+
+        if callable(agent) or hasattr(agent, "invoke"):
+            return self._wrap_callable(agent)
+
+        raise TypeError(
+            f"CortexTracer.wrap() does not support {agent_type}. "
+            "Pass a LangGraph CompiledStateGraph, CrewAI Crew, or any callable."
+        )
+
+    def _wrap_langgraph(self, graph: Any) -> Any:
+        tracer = self
+
+        class InstrumentedGraph:
+            def invoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
+                return tracer._run_traced(
+                    fn=lambda: graph.invoke(input, config, **kwargs),
+                    input=input,
+                    framework="langgraph",
+                )
+
+            async def ainvoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
+                import asyncio
+                return await asyncio.get_event_loop().run_in_executor(
+                    None, lambda: tracer._run_traced(
+                        fn=lambda: graph.invoke(input, config, **kwargs),
+                        input=input,
+                        framework="langgraph",
+                    )
+                )
+
+            def stream(self_, input: dict, config: dict | None = None, **kwargs):
+                return graph.stream(input, config, **kwargs)
+
+            def __getattr__(self_, name: str):
+                return getattr(graph, name)
+
+        return InstrumentedGraph()
+
+    def _wrap_crewai(self, crew: Any) -> Any:
+        tracer = self
+
+        class InstrumentedCrew:
+            def kickoff(self_, inputs: dict | None = None) -> Any:
+                return tracer._run_traced(
+                    fn=lambda: crew.kickoff(inputs=inputs),
+                    input=inputs or {},
+                    framework="crewai",
+                )
+
+            def __getattr__(self_, name: str):
+                return getattr(crew, name)
+
+        return InstrumentedCrew()
+
+    def _wrap_callable(self, fn: Any) -> Any:
+        tracer = self
+
+        if hasattr(fn, "invoke"):
+            # Object with .invoke() — wrap that method
+            original_invoke = fn.invoke
+
+            class InvokeWrapper:
+                def invoke(self_, *args, **kwargs):
+                    input_data = args[0] if args else kwargs
+                    return tracer._run_traced(
+                        fn=lambda: original_invoke(*args, **kwargs),
+                        input=input_data if isinstance(input_data, dict) else {"input": input_data},
+                        framework="generic",
+                    )
+
+                def __getattr__(self_, name: str):
+                    return getattr(fn, name)
+
+            return InvokeWrapper()
+
+        # Plain callable
+        def wrapper(*args, **kwargs):
+            input_data = {"args": list(args), "kwargs": kwargs}
+            return tracer._run_traced(fn=lambda: fn(*args, **kwargs), input=input_data, framework="generic")
+
+        return wrapper
+
+    def _run_traced(self, fn: Callable, input: dict, framework: str) -> Any:
+        trace = Trace(
+            project=self.project,
+            input=input,
+        )
+        self._current_trace = trace
+        t0 = time.perf_counter()
+
+        try:
+            result = fn()
+            trace.total_latency_ms = (time.perf_counter() - t0) * 1000
+            trace.status = RunStatus.COMPLETED
+            trace.output = result if isinstance(result, dict) else {"result": str(result)}
+        except Exception as exc:
+            trace.total_latency_ms = (time.perf_counter() - t0) * 1000
+            trace.status = RunStatus.FAILED
+            trace.failure_kind = FailureKind.UNKNOWN
+            trace.failure_detail = str(exc)
+            raise
+        finally:
+            self._traces.append(trace)
+            if self.api_key:
+                self._flush_trace(trace)
+
+        return result
+
+    @contextmanager
+    def trace_node(self, node_name: str):
+        """Context manager to manually instrument a single node."""
+        node = TraceNode(node_id=str(uuid.uuid4()), node_name=node_name)
+        t0 = time.perf_counter()
+        try:
+            yield node
+        finally:
+            node.latency_ms = (time.perf_counter() - t0) * 1000
+            if self._current_trace:
+                self._current_trace.nodes.append(node)
+
+    def record_tool_call(
+        self,
+        name: str,
+        args: dict | None = None,
+        result: Any = None,
+        error: str | None = None,
+        latency_ms: float = 0.0,
+    ) -> ToolCall:
+        """Manually record a tool call onto the current active trace."""
+        tc = ToolCall(
+            name=name,
+            args=args or {},
+            result=result,
+            status=ToolCallStatus.ERROR if error else ToolCallStatus.SUCCESS,
+            latency_ms=latency_ms,
+            error=error,
+        )
+        if self._current_trace and self._current_trace.nodes:
+            self._current_trace.nodes[-1].tool_calls.append(tc)
+        return tc
+
+    def last_trace(self) -> Trace | None:
+        return self._traces[-1] if self._traces else None
+
+    def traces(self) -> list[Trace]:
+        return list(self._traces)
+
+    def clear(self) -> None:
+        self._traces.clear()
+        self._current_trace = None
+
+    def _flush_trace(self, trace: Trace) -> None:
+        try:
+            import httpx
+            httpx.post(
+                "https://api.cortexops.ai/v1/traces",
+                json=trace.model_dump(mode="json"),
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                timeout=2.0,
+            )
+        except Exception:
+            pass  # non-blocking — tracing never breaks the agent