cortexops 0.1.0__py3-none-any.whl

cortexops/cortexops/cli.py

@@ -0,0 +1,195 @@
+"""CortexOps CLI — cortexops <command> [options]
+
+Commands:
+  eval run     Run an evaluation suite
+  eval diff    Diff two eval runs
+  failures     Show recent failures
+  traces       List recent traces
+  version      Print SDK version
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def cmd_eval_run(args: argparse.Namespace) -> int:
+    """cortexops eval run --dataset golden_v1.yaml --project my-agent"""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+    from cortexops import EvalSuite
+    from cortexops.eval import EvalThresholdError
+
+    print(f"CortexOps eval\n  dataset : {args.dataset}\n  project : {args.project or 'from dataset'}")
+    if args.fail_on:
+        print(f"  fail-on : {args.fail_on}")
+    print()
+
+    def passthrough_agent(inp: dict) -> dict:
+        """Placeholder — replace with your actual agent import."""
+        return {"output": f"[no agent bound] input was: {inp}"}
+
+    try:
+        agent = _load_agent(args.agent) if args.agent else passthrough_agent
+        summary = EvalSuite.run(
+            dataset=args.dataset,
+            agent=agent,
+            verbose=not args.quiet,
+            fail_on=args.fail_on,
+        )
+    except EvalThresholdError as e:
+        print(f"\nCI gate FAILED: {e}", file=sys.stderr)
+        return 1
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    if args.output:
+        Path(args.output).write_text(json.dumps(summary.model_dump(mode="json"), indent=2))
+        print(f"\nResults written to {args.output}")
+
+    return 0 if summary.failed == 0 else 1
+
+
+def cmd_eval_diff(args: argparse.Namespace) -> int:
+    """cortexops eval diff <run_a> <run_b> --api-key cxo-..."""
+    from cortexops import CortexClient
+
+    api_key = args.api_key or os.getenv("CORTEXOPS_API_KEY")
+    if not api_key:
+        print("Error: --api-key or CORTEXOPS_API_KEY required for diff", file=sys.stderr)
+        return 1
+
+    client = CortexClient(api_key=api_key, base_url=args.base_url)
+    try:
+        diff = client.diff(args.run_a, args.run_b)
+    except Exception as e:
+        print(f"Error fetching diff: {e}", file=sys.stderr)
+        return 1
+
+    delta_tc = diff.get("task_completion_delta", 0)
+    delta_tool = diff.get("tool_accuracy_delta", 0)
+    regressions = diff.get("regressions", [])
+    improvements = diff.get("improvements", [])
+
+    sign = lambda v: f"+{v:.1%}" if v >= 0 else f"{v:.1%}"
+    print(f"Diff: {args.run_a[:8]} → {args.run_b[:8]}")
+    print(f"  Task completion : {sign(delta_tc)}")
+    print(f"  Tool accuracy   : {sign(delta_tool / 100)}")
+    if regressions:
+        print(f"  Regressions ({len(regressions)}): {', '.join(regressions)}")
+    if improvements:
+        print(f"  Improvements ({len(improvements)}): {', '.join(improvements)}")
+
+    return 1 if regressions else 0
+
+
+def cmd_failures(args: argparse.Namespace) -> int:
+    """cortexops failures --project my-agent --last 24h"""
+    from cortexops import CortexClient
+
+    api_key = args.api_key or os.getenv("CORTEXOPS_API_KEY")
+    if not api_key:
+        print("Error: --api-key or CORTEXOPS_API_KEY required", file=sys.stderr)
+        return 1
+
+    client = CortexClient(api_key=api_key, base_url=args.base_url)
+    try:
+        traces = client.list_traces(project=args.project, limit=args.limit)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    failed = [t for t in traces if t.get("status") == "failed"]
+    if not failed:
+        print(f"No failures found for project '{args.project}'")
+        return 0
+
+    print(f"Failures — {args.project} (last {len(traces)} traces)")
+    print(f"{'Trace ID':<36}  {'Failure kind':<28}  Latency")
+    print("-" * 78)
+    for t in failed[:args.limit]:
+        print(
+            f"{t['trace_id']:<36}  {t.get('failure_kind') or 'unknown':<28}  "
+            f"{t.get('total_latency_ms', 0):.0f}ms"
+        )
+    return 0
+
+
+def cmd_version(_: argparse.Namespace) -> int:
+    from cortexops import __version__
+    print(f"cortexops {__version__}")
+    return 0
+
+
+def _load_agent(agent_path: str):
+    """Load an agent from a dotted path like 'mymodule:my_agent'."""
+    if ":" not in agent_path:
+        print(f"Error: --agent must be in the format 'module:object', got '{agent_path}'", file=sys.stderr)
+        sys.exit(1)
+    module_path, attr = agent_path.rsplit(":", 1)
+    import importlib
+    module = importlib.import_module(module_path)
+    return getattr(module, attr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="cortexops",
+        description="CortexOps — reliability infrastructure for AI agents",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    # ── eval ──────────────────────────────────────────────────────────────
+    eval_parser = sub.add_parser("eval", help="Evaluation commands")
+    eval_sub = eval_parser.add_subparsers(dest="eval_command")
+
+    run_p = eval_sub.add_parser("run", help="Run an eval suite")
+    run_p.add_argument("--dataset", "-d", required=True, help="Path to golden dataset YAML")
+    run_p.add_argument("--project", "-p", default=None, help="Project name (overrides dataset)")
+    run_p.add_argument("--agent", "-a", default=None, help="Agent to evaluate (module:object)")
+    run_p.add_argument("--fail-on", default=None, help="e.g. 'task_completion < 0.90'")
+    run_p.add_argument("--output", "-o", default=None, help="Save JSON results to file")
+    run_p.add_argument("--quiet", "-q", action="store_true", help="Suppress per-case output")
+
+    diff_p = eval_sub.add_parser("diff", help="Diff two eval runs")
+    diff_p.add_argument("run_a", help="First run ID")
+    diff_p.add_argument("run_b", help="Second run ID")
+    diff_p.add_argument("--api-key", default=None)
+    diff_p.add_argument("--base-url", default="https://api.cortexops.ai")
+
+    # ── failures ──────────────────────────────────────────────────────────
+    fail_p = sub.add_parser("failures", help="List recent agent failures")
+    fail_p.add_argument("--project", "-p", required=True)
+    fail_p.add_argument("--limit", "-n", type=int, default=20)
+    fail_p.add_argument("--api-key", default=None)
+    fail_p.add_argument("--base-url", default="https://api.cortexops.ai")
+
+    # ── version ───────────────────────────────────────────────────────────
+    sub.add_parser("version", help="Print version and exit")
+
+    args = parser.parse_args()
+
+    handlers = {
+        ("eval", "run"): cmd_eval_run,
+        ("eval", "diff"): cmd_eval_diff,
+        ("failures", None): cmd_failures,
+        ("version", None): cmd_version,
+    }
+
+    key = (args.command, getattr(args, "eval_command", None))
+    handler = handlers.get(key)
+
+    if handler is None:
+        parser.print_help()
+        sys.exit(0)
+
+    sys.exit(handler(args))
+
+
+if __name__ == "__main__":
+    main()

cortexops/cortexops/client.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urljoin
+
+from .models import EvalSummary, Trace
+
+
+class CortexClient:
+    """HTTP client for the CortexOps backend API.
+
+    Used by the SDK to push traces and pull eval history.
+    Not required for local-only usage.
+
+    Usage:
+        client = CortexClient(api_key="cxo-...", base_url="https://api.cortexops.ai")
+        client.push_trace(tracer.last_trace())
+        history = client.list_runs(project="payments-agent", limit=10)
+    """
+
+    DEFAULT_BASE_URL = "https://api.cortexops.ai"
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 10.0,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = (base_url or self.DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = timeout
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _get(self, path: str, params: dict | None = None) -> dict:
+        import httpx
+        r = httpx.get(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            params=params,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def _post(self, path: str, data: dict) -> dict:
+        import httpx
+        r = httpx.post(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            json=data,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def push_trace(self, trace: Trace) -> dict:
+        return self._post("/v1/traces", trace.model_dump(mode="json"))
+
+    def get_trace(self, trace_id: str) -> dict:
+        return self._get(f"/v1/traces/{trace_id}")
+
+    def list_traces(self, project: str, limit: int = 50) -> list[dict]:
+        return self._get("/v1/traces", {"project": project, "limit": limit})
+
+    def push_eval(self, summary: EvalSummary) -> dict:
+        return self._post("/v1/evals", summary.model_dump(mode="json"))
+
+    def list_runs(self, project: str, limit: int = 10) -> list[dict]:
+        return self._get("/v1/evals", {"project": project, "limit": limit})
+
+    def run_eval(self, dataset: str, project: str) -> dict:
+        """Trigger a server-side eval run (async via Celery)."""
+        return self._post("/v1/evals/run", {"dataset": dataset, "project": project})
+
+    def get_eval(self, run_id: str) -> dict:
+        return self._get(f"/v1/evals/{run_id}")
+
+    def diff(self, run_id_a: str, run_id_b: str) -> dict:
+        return self._get("/v1/evals/diff", {"a": run_id_a, "b": run_id_b})

cortexops/cortexops/eval.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import statistics
+import time
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .metrics import compute_case_result
+from .models import (
+    EvalCase,
+    EvalDataset,
+    EvalSummary,
+    FailureKind,
+    RunStatus,
+    Trace,
+    TraceNode,
+)
+
+
+class EvalSuite:
+    """Run evaluation suites against any instrumented agent.
+
+    Usage:
+        results = EvalSuite.run(
+            dataset="golden_v1.yaml",
+            agent=your_langgraph_app,   # wrapped or raw callable
+        )
+        print(results.summary())
+    """
+
+    @classmethod
+    def run(
+        cls,
+        dataset: str | Path | dict | EvalDataset,
+        agent: Any,
+        *,
+        metrics: list[str] | None = None,
+        verbose: bool = True,
+        fail_on: str | None = None,
+    ) -> EvalSummary:
+        """Run a full eval suite.
+
+        Args:
+            dataset:  Path to YAML, dict, or EvalDataset object.
+            agent:    Any callable that accepts a string or dict input.
+            metrics:  Optional subset of metrics to run.
+            verbose:  Print case-by-case progress.
+            fail_on:  Threshold expression like "task_completion < 0.90".
+                      Raises EvalThresholdError if the condition is met.
+        """
+        ds = cls._load_dataset(dataset)
+        case_results = []
+
+        for i, case in enumerate(ds.cases):
+            if verbose:
+                print(f"  [{i+1}/{len(ds.cases)}] {case.id} ... ", end="", flush=True)
+
+            trace = cls._run_case(agent, case)
+            result = compute_case_result(case, trace)
+            case_results.append(result)
+
+            if verbose:
+                status = "pass" if result.passed else "FAIL"
+                print(f"{status} ({result.score:.0f})")
+
+        latencies = [r.latency_ms for r in case_results]
+        latencies_sorted = sorted(latencies)
+        n = len(latencies_sorted)
+
+        summary = EvalSummary(
+            project=ds.project,
+            dataset_version=ds.version,
+            total_cases=len(case_results),
+            passed=sum(1 for r in case_results if r.passed),
+            failed=sum(1 for r in case_results if not r.passed),
+            warnings=sum(1 for r in case_results if not r.passed and r.score >= 60),
+            task_completion_rate=sum(1 for r in case_results if r.task_completion) / max(n, 1),
+            tool_accuracy=statistics.mean(r.tool_accuracy for r in case_results) if case_results else 0.0,
+            latency_p50_ms=latencies_sorted[int(n * 0.50) - 1] if n else 0.0,
+            latency_p95_ms=latencies_sorted[int(n * 0.95) - 1] if n else 0.0,
+            case_results=case_results,
+        )
+
+        if verbose:
+            print()
+            print(summary.summary())
+
+        if fail_on:
+            cls._check_threshold(summary, fail_on)
+
+        return summary
+
+    @classmethod
+    def _run_case(cls, agent: Any, case: EvalCase) -> Trace:
+        input_data = case.input if isinstance(case.input, dict) else {"input": case.input}
+        t0 = time.perf_counter()
+
+        try:
+            if hasattr(agent, "invoke"):
+                output = agent.invoke(input_data)
+            elif callable(agent):
+                output = agent(input_data)
+            else:
+                raise TypeError(f"Agent {type(agent).__name__} is not callable")
+            latency_ms = (time.perf_counter() - t0) * 1000
+            output_dict = output if isinstance(output, dict) else {"output": str(output)}
+
+            return Trace(
+                project="eval",
+                case_id=case.id,
+                input=input_data,
+                output=output_dict,
+                total_latency_ms=latency_ms,
+                status=RunStatus.COMPLETED,
+                nodes=[
+                    TraceNode(
+                        node_id="eval_root",
+                        node_name="agent",
+                        input=input_data,
+                        output=output_dict,
+                        latency_ms=latency_ms,
+                    )
+                ],
+            )
+
+        except Exception as exc:
+            latency_ms = (time.perf_counter() - t0) * 1000
+            return Trace(
+                project="eval",
+                case_id=case.id,
+                input=input_data,
+                output={},
+                total_latency_ms=latency_ms,
+                status=RunStatus.FAILED,
+                failure_kind=FailureKind.UNKNOWN,
+                failure_detail=str(exc),
+            )
+
+    @classmethod
+    def _load_dataset(cls, dataset: str | Path | dict | EvalDataset) -> EvalDataset:
+        if isinstance(dataset, EvalDataset):
+            return dataset
+
+        if isinstance(dataset, dict):
+            return cls._parse_dataset_dict(dataset)
+
+        path = Path(dataset)
+        if not path.exists():
+            raise FileNotFoundError(f"Dataset not found: {path}")
+
+        raw = yaml.safe_load(path.read_text())
+        return cls._parse_dataset_dict(raw)
+
+    @classmethod
+    def _parse_dataset_dict(cls, raw: dict) -> EvalDataset:
+        cases = []
+        for c in raw.get("cases", []):
+            cases.append(
+                EvalCase(
+                    id=str(c["id"]),
+                    input=c["input"],
+                    expected_tool_calls=c.get("expected_tool_calls", []),
+                    expected_output_contains=c.get("expected_output_contains", []),
+                    expected_output_not_contains=c.get("expected_output_not_contains", []),
+                    max_latency_ms=c.get("max_latency_ms"),
+                    judge=c.get("judge", "rule"),
+                    judge_criteria=c.get("judge_criteria"),
+                    tags=c.get("tags", []),
+                )
+            )
+        return EvalDataset(
+            version=raw.get("version", 1),
+            project=raw.get("project", "unknown"),
+            description=raw.get("description", ""),
+            cases=cases,
+        )
+
+    @classmethod
+    def _check_threshold(cls, summary: EvalSummary, fail_on: str) -> None:
+        """Parse and evaluate a threshold expression like 'task_completion < 0.90'."""
+        import re
+
+        m = re.match(r"(\w+)\s*([<>]=?)\s*([\d.]+)", fail_on.strip())
+        if not m:
+            raise ValueError(f"Invalid fail_on expression: '{fail_on}'")
+
+        metric, op, threshold_str = m.groups()
+        threshold = float(threshold_str)
+
+        actual = {
+            "task_completion": summary.task_completion_rate,
+            "tool_accuracy": summary.tool_accuracy / 100.0,
+            "pass_rate": summary.passed / max(summary.total_cases, 1),
+        }.get(metric)
+
+        if actual is None:
+            raise ValueError(f"Unknown metric in fail_on: '{metric}'")
+
+        failed = {
+            "<": actual < threshold,
+            "<=": actual <= threshold,
+            ">": actual > threshold,
+            ">=": actual >= threshold,
+        }.get(op, False)
+
+        if failed:
+            raise EvalThresholdError(
+                f"Eval gate failed: {metric}={actual:.3f} {op} {threshold} "
+                f"(project={summary.project})"
+            )
+
+
+class EvalThresholdError(Exception):
+    """Raised when an eval run fails a CI threshold gate."""

cortexops/cortexops/judge.py

@@ -0,0 +1,155 @@
+"""LLM-as-judge metric for CortexOps.
+
+Uses an LLM (default: gpt-4o-mini) to score open-ended agent output
+against natural language criteria. Works with any OpenAI-compatible API.
+
+Usage in golden dataset:
+    - id: refund_explanation_01
+      input: "Why was my refund rejected?"
+      judge: llm
+      judge_criteria: >
+        The response should explain the rejection reason clearly,
+        be empathetic, and offer a next step to the customer.
+        It must NOT contain jargon or mention internal system errors.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from .metrics import Metric
+from .models import EvalCase, FailureKind, Trace
+
+
+JUDGE_SYSTEM_PROMPT = """You are a strict but fair evaluator of AI agent outputs.
+You will be given:
+  - The user's input to the agent
+  - The agent's output
+  - Evaluation criteria
+
+Score the output from 0 to 100 and explain your reasoning briefly.
+
+Respond ONLY with valid JSON in this exact format:
+{
+  "score": <integer 0-100>,
+  "passed": <true|false>,
+  "reasoning": "<one sentence>"
+}
+
+Rules:
+- 90-100: Fully meets all criteria, no issues
+- 70-89: Mostly meets criteria, minor gaps
+- 50-69: Partially meets criteria, notable gaps
+- 0-49: Fails to meet criteria or contains harmful/incorrect content
+- passed = true only if score >= 70
+"""
+
+
+class LLMJudgeMetric(Metric):
+    """Score agent output using an LLM judge.
+
+    Falls back to a heuristic score if the LLM API is unavailable,
+    so evals never block on API failures.
+
+    Args:
+        model:          OpenAI model to use. Default: gpt-4o-mini.
+        api_key:        OpenAI API key. Falls back to OPENAI_API_KEY env var.
+        base_url:       OpenAI-compatible base URL. Useful for local LLMs.
+        temperature:    Judge temperature. Keep low (0.1) for consistency.
+        timeout:        HTTP timeout in seconds.
+    """
+
+    name = "llm_judge"
+
+    def __init__(
+        self,
+        model: str = "gpt-4o-mini",
+        api_key: str | None = None,
+        base_url: str = "https://api.openai.com/v1",
+        temperature: float = 0.1,
+        timeout: float = 30.0,
+    ) -> None:
+        self.model = model
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY") or os.getenv("CORTEXOPS_JUDGE_API_KEY")
+        self.base_url = base_url.rstrip("/")
+        self.temperature = temperature
+        self.timeout = timeout
+
+    def score(self, case: EvalCase, trace: Trace) -> tuple[float, FailureKind | None, str | None]:
+        if not case.judge_criteria:
+            return 100.0, None, None
+
+        if case.judge != "llm":
+            return 100.0, None, None
+
+        user_input = str(case.input)
+        agent_output = str(trace.output.get("output", trace.output))
+
+        try:
+            result = self._call_judge(user_input, agent_output, case.judge_criteria)
+            score = float(result.get("score", 0))
+            passed = result.get("passed", score >= 70)
+            reasoning = result.get("reasoning", "")
+
+            if not passed:
+                return score, FailureKind.OUTPUT_FORMAT, f"LLM judge: {reasoning}"
+            return score, None, None
+
+        except Exception as exc:
+            return self._heuristic_fallback(case, trace, str(exc))
+
+    def _call_judge(self, user_input: str, agent_output: str, criteria: str) -> dict[str, Any]:
+        import httpx
+
+        if not self.api_key:
+            raise ValueError(
+                "No API key found for LLM judge. Set OPENAI_API_KEY or pass api_key= to LLMJudgeMetric()."
+            )
+
+        user_message = (
+            f"USER INPUT:\n{user_input}\n\n"
+            f"AGENT OUTPUT:\n{agent_output}\n\n"
+            f"EVALUATION CRITERIA:\n{criteria}"
+        )
+
+        response = httpx.post(
+            f"{self.base_url}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            },
+            json={
+                "model": self.model,
+                "messages": [
+                    {"role": "system", "content": JUDGE_SYSTEM_PROMPT},
+                    {"role": "user", "content": user_message},
+                ],
+                "temperature": self.temperature,
+                "response_format": {"type": "json_object"},
+            },
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+        return json.loads(content)
+
+    def _heuristic_fallback(
+        self, case: EvalCase, trace: Trace, error: str
+    ) -> tuple[float, FailureKind | None, str | None]:
+        """Simple keyword fallback when the LLM is unavailable."""
+        output = str(trace.output.get("output", "")).lower()
+        criteria_words = (case.judge_criteria or "").lower().split()
+        meaningful_words = [w for w in criteria_words if len(w) > 4]
+
+        if not meaningful_words:
+            return 70.0, None, f"LLM judge unavailable ({error[:60]}); heuristic used"
+
+        hits = sum(1 for w in meaningful_words if w in output)
+        ratio = hits / len(meaningful_words)
+        score = 50.0 + 50.0 * ratio
+
+        if score < 70:
+            return score, FailureKind.OUTPUT_FORMAT, f"LLM judge unavailable; heuristic score {score:.0f}"
+        return score, None, f"LLM judge unavailable ({error[:60]}); heuristic used"