cortexops 0.1.0__py3-none-any.whl

cortexops/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 CortexOps Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cortexops/README.md

@@ -0,0 +1,106 @@
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/ashishodu2023/cortexops/blob/main/LICENSE)
+
+---
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Install
+
+```bash
+pip install cortexops
+
+# With HTTP client (for pushing traces to hosted API):
+pip install cortexops[http]
+
+# With LLM judge support:
+pip install cortexops[llm]
+```
+
+---
+
+## Quickstart
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+print(results.summary())
+```
+
+---
+
+## Golden dataset (YAML)
+
+```yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: open_ended_explanation_01
+    input: "Why was my refund rejected?"
+    judge: llm
+    judge_criteria: >
+      The response must explain the rejection reason clearly,
+      be empathetic, and offer a concrete next step. No jargon.
+```
+
+---
+
+## CI gate
+
+```bash
+cortexops eval run \
+  --dataset golden_v1.yaml \
+  --fail-on "task_completion < 0.90"
+```
+
+Exits non-zero if the threshold is not met — blocks the PR.
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Non-empty, non-error output with expected content |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Fabrication signals in output |
+| `llm_judge` | GPT-4o scores against natural-language criteria |
+
+---
+
+## Links
+
+- **Docs**: [docs.cortexops.ai](https://docs.cortexops.ai)
+- **Repo**: [github.com/ashishodu2023/cortexops](https://github.com/ashishodu2023/cortexops)
+- **Issues**: [GitHub Issues](https://github.com/ashishodu2023/cortexops/issues)

cortexops/__init__.py

@@ -0,0 +1,58 @@
+"""CortexOps — Reliability infrastructure for AI agents.
+
+Quickstart:
+    from cortexops import CortexTracer, EvalSuite
+
+    tracer = CortexTracer(project="my-agent")
+    graph  = tracer.wrap(your_langgraph_app)
+
+    results = EvalSuite.run(dataset="golden_v1.yaml", agent=graph)
+    print(results.summary())
+"""
+
+from .client import CortexClient
+from .eval import EvalSuite, EvalThresholdError
+from .judge import LLMJudgeMetric
+from .metrics import (
+    HallucinationMetric,
+    LatencyMetric,
+    Metric,
+    TaskCompletionMetric,
+    ToolAccuracyMetric,
+)
+from .models import (
+    CaseResult,
+    EvalCase,
+    EvalDataset,
+    EvalSummary,
+    FailureKind,
+    RunStatus,
+    Trace,
+    TraceNode,
+    ToolCall,
+)
+from .tracer import CortexTracer
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CortexTracer",
+    "EvalSuite",
+    "EvalThresholdError",
+    "CortexClient",
+    "Metric",
+    "TaskCompletionMetric",
+    "ToolAccuracyMetric",
+    "LatencyMetric",
+    "HallucinationMetric",
+    "LLMJudgeMetric",
+    "Trace",
+    "TraceNode",
+    "ToolCall",
+    "EvalCase",
+    "EvalDataset",
+    "EvalSummary",
+    "CaseResult",
+    "FailureKind",
+    "RunStatus",
+]

cortexops/cli.py

@@ -0,0 +1,195 @@
+"""CortexOps CLI — cortexops <command> [options]
+
+Commands:
+  eval run     Run an evaluation suite
+  eval diff    Diff two eval runs
+  failures     Show recent failures
+  traces       List recent traces
+  version      Print SDK version
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def cmd_eval_run(args: argparse.Namespace) -> int:
+    """cortexops eval run --dataset golden_v1.yaml --project my-agent"""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+    from cortexops import EvalSuite
+    from cortexops.eval import EvalThresholdError
+
+    print(f"CortexOps eval\n  dataset : {args.dataset}\n  project : {args.project or 'from dataset'}")
+    if args.fail_on:
+        print(f"  fail-on : {args.fail_on}")
+    print()
+
+    def passthrough_agent(inp: dict) -> dict:
+        """Placeholder — replace with your actual agent import."""
+        return {"output": f"[no agent bound] input was: {inp}"}
+
+    try:
+        agent = _load_agent(args.agent) if args.agent else passthrough_agent
+        summary = EvalSuite.run(
+            dataset=args.dataset,
+            agent=agent,
+            verbose=not args.quiet,
+            fail_on=args.fail_on,
+        )
+    except EvalThresholdError as e:
+        print(f"\nCI gate FAILED: {e}", file=sys.stderr)
+        return 1
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    if args.output:
+        Path(args.output).write_text(json.dumps(summary.model_dump(mode="json"), indent=2))
+        print(f"\nResults written to {args.output}")
+
+    return 0 if summary.failed == 0 else 1
+
+
+def cmd_eval_diff(args: argparse.Namespace) -> int:
+    """cortexops eval diff <run_a> <run_b> --api-key cxo-..."""
+    from cortexops import CortexClient
+
+    api_key = args.api_key or os.getenv("CORTEXOPS_API_KEY")
+    if not api_key:
+        print("Error: --api-key or CORTEXOPS_API_KEY required for diff", file=sys.stderr)
+        return 1
+
+    client = CortexClient(api_key=api_key, base_url=args.base_url)
+    try:
+        diff = client.diff(args.run_a, args.run_b)
+    except Exception as e:
+        print(f"Error fetching diff: {e}", file=sys.stderr)
+        return 1
+
+    delta_tc = diff.get("task_completion_delta", 0)
+    delta_tool = diff.get("tool_accuracy_delta", 0)
+    regressions = diff.get("regressions", [])
+    improvements = diff.get("improvements", [])
+
+    sign = lambda v: f"+{v:.1%}" if v >= 0 else f"{v:.1%}"
+    print(f"Diff: {args.run_a[:8]} → {args.run_b[:8]}")
+    print(f"  Task completion : {sign(delta_tc)}")
+    print(f"  Tool accuracy   : {sign(delta_tool / 100)}")
+    if regressions:
+        print(f"  Regressions ({len(regressions)}): {', '.join(regressions)}")
+    if improvements:
+        print(f"  Improvements ({len(improvements)}): {', '.join(improvements)}")
+
+    return 1 if regressions else 0
+
+
+def cmd_failures(args: argparse.Namespace) -> int:
+    """cortexops failures --project my-agent --last 24h"""
+    from cortexops import CortexClient
+
+    api_key = args.api_key or os.getenv("CORTEXOPS_API_KEY")
+    if not api_key:
+        print("Error: --api-key or CORTEXOPS_API_KEY required", file=sys.stderr)
+        return 1
+
+    client = CortexClient(api_key=api_key, base_url=args.base_url)
+    try:
+        traces = client.list_traces(project=args.project, limit=args.limit)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    failed = [t for t in traces if t.get("status") == "failed"]
+    if not failed:
+        print(f"No failures found for project '{args.project}'")
+        return 0
+
+    print(f"Failures — {args.project} (last {len(traces)} traces)")
+    print(f"{'Trace ID':<36}  {'Failure kind':<28}  Latency")
+    print("-" * 78)
+    for t in failed[:args.limit]:
+        print(
+            f"{t['trace_id']:<36}  {t.get('failure_kind') or 'unknown':<28}  "
+            f"{t.get('total_latency_ms', 0):.0f}ms"
+        )
+    return 0
+
+
+def cmd_version(_: argparse.Namespace) -> int:
+    from cortexops import __version__
+    print(f"cortexops {__version__}")
+    return 0
+
+
+def _load_agent(agent_path: str):
+    """Load an agent from a dotted path like 'mymodule:my_agent'."""
+    if ":" not in agent_path:
+        print(f"Error: --agent must be in the format 'module:object', got '{agent_path}'", file=sys.stderr)
+        sys.exit(1)
+    module_path, attr = agent_path.rsplit(":", 1)
+    import importlib
+    module = importlib.import_module(module_path)
+    return getattr(module, attr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="cortexops",
+        description="CortexOps — reliability infrastructure for AI agents",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    # ── eval ──────────────────────────────────────────────────────────────
+    eval_parser = sub.add_parser("eval", help="Evaluation commands")
+    eval_sub = eval_parser.add_subparsers(dest="eval_command")
+
+    run_p = eval_sub.add_parser("run", help="Run an eval suite")
+    run_p.add_argument("--dataset", "-d", required=True, help="Path to golden dataset YAML")
+    run_p.add_argument("--project", "-p", default=None, help="Project name (overrides dataset)")
+    run_p.add_argument("--agent", "-a", default=None, help="Agent to evaluate (module:object)")
+    run_p.add_argument("--fail-on", default=None, help="e.g. 'task_completion < 0.90'")
+    run_p.add_argument("--output", "-o", default=None, help="Save JSON results to file")
+    run_p.add_argument("--quiet", "-q", action="store_true", help="Suppress per-case output")
+
+    diff_p = eval_sub.add_parser("diff", help="Diff two eval runs")
+    diff_p.add_argument("run_a", help="First run ID")
+    diff_p.add_argument("run_b", help="Second run ID")
+    diff_p.add_argument("--api-key", default=None)
+    diff_p.add_argument("--base-url", default="https://api.cortexops.ai")
+
+    # ── failures ──────────────────────────────────────────────────────────
+    fail_p = sub.add_parser("failures", help="List recent agent failures")
+    fail_p.add_argument("--project", "-p", required=True)
+    fail_p.add_argument("--limit", "-n", type=int, default=20)
+    fail_p.add_argument("--api-key", default=None)
+    fail_p.add_argument("--base-url", default="https://api.cortexops.ai")
+
+    # ── version ───────────────────────────────────────────────────────────
+    sub.add_parser("version", help="Print version and exit")
+
+    args = parser.parse_args()
+
+    handlers = {
+        ("eval", "run"): cmd_eval_run,
+        ("eval", "diff"): cmd_eval_diff,
+        ("failures", None): cmd_failures,
+        ("version", None): cmd_version,
+    }
+
+    key = (args.command, getattr(args, "eval_command", None))
+    handler = handlers.get(key)
+
+    if handler is None:
+        parser.print_help()
+        sys.exit(0)
+
+    sys.exit(handler(args))
+
+
+if __name__ == "__main__":
+    main()

cortexops/client.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urljoin
+
+from .models import EvalSummary, Trace
+
+
+class CortexClient:
+    """HTTP client for the CortexOps backend API.
+
+    Used by the SDK to push traces and pull eval history.
+    Not required for local-only usage.
+
+    Usage:
+        client = CortexClient(api_key="cxo-...", base_url="https://api.cortexops.ai")
+        client.push_trace(tracer.last_trace())
+        history = client.list_runs(project="payments-agent", limit=10)
+    """
+
+    DEFAULT_BASE_URL = "https://api.cortexops.ai"
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 10.0,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = (base_url or self.DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = timeout
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _get(self, path: str, params: dict | None = None) -> dict:
+        import httpx
+        r = httpx.get(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            params=params,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def _post(self, path: str, data: dict) -> dict:
+        import httpx
+        r = httpx.post(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            json=data,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def push_trace(self, trace: Trace) -> dict:
+        return self._post("/v1/traces", trace.model_dump(mode="json"))
+
+    def get_trace(self, trace_id: str) -> dict:
+        return self._get(f"/v1/traces/{trace_id}")
+
+    def list_traces(self, project: str, limit: int = 50) -> list[dict]:
+        return self._get("/v1/traces", {"project": project, "limit": limit})
+
+    def push_eval(self, summary: EvalSummary) -> dict:
+        return self._post("/v1/evals", summary.model_dump(mode="json"))
+
+    def list_runs(self, project: str, limit: int = 10) -> list[dict]:
+        return self._get("/v1/evals", {"project": project, "limit": limit})
+
+    def run_eval(self, dataset: str, project: str) -> dict:
+        """Trigger a server-side eval run (async via Celery)."""
+        return self._post("/v1/evals/run", {"dataset": dataset, "project": project})
+
+    def get_eval(self, run_id: str) -> dict:
+        return self._get(f"/v1/evals/{run_id}")
+
+    def diff(self, run_id_a: str, run_id_b: str) -> dict:
+        return self._get("/v1/evals/diff", {"a": run_id_a, "b": run_id_b})

cortexops/cortexops/__init__.py

@@ -0,0 +1,58 @@
+"""CortexOps — Reliability infrastructure for AI agents.
+
+Quickstart:
+    from cortexops import CortexTracer, EvalSuite
+
+    tracer = CortexTracer(project="my-agent")
+    graph  = tracer.wrap(your_langgraph_app)
+
+    results = EvalSuite.run(dataset="golden_v1.yaml", agent=graph)
+    print(results.summary())
+"""
+
+from .client import CortexClient
+from .eval import EvalSuite, EvalThresholdError
+from .judge import LLMJudgeMetric
+from .metrics import (
+    HallucinationMetric,
+    LatencyMetric,
+    Metric,
+    TaskCompletionMetric,
+    ToolAccuracyMetric,
+)
+from .models import (
+    CaseResult,
+    EvalCase,
+    EvalDataset,
+    EvalSummary,
+    FailureKind,
+    RunStatus,
+    Trace,
+    TraceNode,
+    ToolCall,
+)
+from .tracer import CortexTracer
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CortexTracer",
+    "EvalSuite",
+    "EvalThresholdError",
+    "CortexClient",
+    "Metric",
+    "TaskCompletionMetric",
+    "ToolAccuracyMetric",
+    "LatencyMetric",
+    "HallucinationMetric",
+    "LLMJudgeMetric",
+    "Trace",
+    "TraceNode",
+    "ToolCall",
+    "EvalCase",
+    "EvalDataset",
+    "EvalSummary",
+    "CaseResult",
+    "FailureKind",
+    "RunStatus",
+]