cortexops 0.4.0__tar.gz → 0.5.0__tar.gz

{cortexops-0.4.0 → cortexops-0.5.0}/.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *.pyo
+.venv-e2e/
 .venv/
 venv/
 .env

{cortexops-0.4.0 → cortexops-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: cortexops
-Version: 0.4.0
-Summary: Reliability infrastructure for AI agents — evaluation, observability, and regression testing
+Version: 0.5.0
+Summary: AI agent observability & eval platform — trace, monitor, and eval-gate LangGraph, CrewAI, OpenAI Agents, PydanticAI, Agno, AutoGen, Haystack, DSPy, LlamaIndex, Smolagents, and Google ADK
 Project-URL: Homepage, https://getcortexops.com
 Project-URL: Repository, https://github.com/ashishodu2023/cortexops
 Project-URL: Documentation, https://docs.getcortexops.com
@@ -44,6 +44,7 @@ Classifier: Topic :: Software Development :: Quality Assurance
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Provides-Extra: all

{cortexops-0.4.0 → cortexops-0.5.0}/cortexops/__init__.py

@@ -34,7 +34,7 @@ from .models import (
 )
 from .tracer import CortexTracer
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 
 __all__ = [
     "CortexTracer",

cortexops-0.5.0/cortexops/client.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from urllib.parse import urljoin
+
+from .models import Trace
+
+
+class CortexClient:
+    """HTTP client for the CortexOps backend API.
+
+    Used by the SDK to push traces and pull eval history.
+    Not required for local-only usage.
+
+    Usage:
+        client = CortexClient(api_key="cxo-...", base_url="https://api.cortexops.ai")
+        client.push_trace(tracer.last_trace())
+        history = client.list_runs(project="payments-agent", limit=10)
+    """
+
+    DEFAULT_BASE_URL = "https://api.getcortexops.com"
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 10.0,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = (base_url or self.DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = timeout
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "X-API-Key": self.api_key,
+            "Content-Type": "application/json",
+        }
+
+    def _get(self, path: str, params: dict | None = None) -> dict:
+        import httpx
+        r = httpx.get(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            params=params,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def _post(self, path: str, data: dict) -> dict:
+        import httpx
+        r = httpx.post(
+            urljoin(self.base_url + "/", path.lstrip("/")),
+            headers=self._headers(),
+            json=data,
+            timeout=self.timeout,
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def push_trace(self, trace: Trace) -> dict:
+        return self._post("/v1/traces", trace.model_dump(mode="json"))
+
+    def get_trace(self, trace_id: str) -> dict:
+        return self._get(f"/v1/traces/{trace_id}")
+
+    def list_traces(self, project: str, limit: int = 50) -> list[dict]:
+        return self._get("/v1/traces", {"project": project, "limit": limit})
+
+    def push_eval(self, summary, *, dataset: dict | None = None, prompt: dict | None = None) -> dict:
+        """Push a completed EvalSuite summary to POST /v1/evals/ingest."""
+        if hasattr(summary, "model_dump"):
+            data = summary.model_dump(mode="json")
+        else:
+            data = dict(summary)
+        for cr in data.get("case_results", []):
+            cr.pop("trace", None)
+            if cr.get("failure_kind") and hasattr(cr["failure_kind"], "value"):
+                cr["failure_kind"] = cr["failure_kind"].value
+        if dataset:
+            data["dataset"] = dataset
+        if prompt:
+            data["prompt"] = prompt
+        return self._post("/v1/evals/ingest", data)
+
+    def project_for_key(self) -> str:
+        """Return the project bound to the current API key."""
+        return self._get("/v1/traces/quota")["project"]
+
+    def list_runs(self, project: str, limit: int = 10) -> list[dict]:
+        return self._get("/v1/evals", {"project": project, "limit": limit})
+
+    def run_eval(self, dataset: str, project: str) -> dict:
+        """Trigger a server-side eval run (async via Celery)."""
+        return self._post("/v1/evals/run", {"dataset": dataset, "project": project})
+
+    def get_eval(self, run_id: str) -> dict:
+        return self._get(f"/v1/evals/{run_id}")
+
+    def diff(self, run_id_a: str, run_id_b: str) -> dict:
+        return self._get("/v1/evals/diff", {"a": run_id_a, "b": run_id_b})
+
+    def list_datasets(self) -> list[dict]:
+        return self._get("/v1/eval/datasets")
+
+    def get_dataset(self, dataset_id: str) -> dict:
+        return self._get(f"/v1/eval/datasets/{dataset_id}")
+
+    def create_dataset(self, name: str, cases: list[dict], description: str = "") -> dict:
+        return self._post("/v1/eval/datasets", {
+            "name": name,
+            "description": description,
+            "cases": cases,
+        })
+
+    def add_case_from_trace(
+        self,
+        dataset_id: str,
+        trace_id: str,
+        *,
+        case_id: str | None = None,
+    ) -> dict:
+        """Promote a production trace into a golden dataset case."""
+        body: dict = {"trace_id": trace_id}
+        if case_id:
+            body["case_id"] = case_id
+        return self._post(f"/v1/eval/datasets/{dataset_id}/cases/from-trace", body)
+
+    def commit_prompt(
+        self,
+        project: str,
+        prompt_name: str,
+        content: str,
+        *,
+        model: str = "",
+        temperature: float = 0.7,
+        commit_message: str = "",
+        author: str = "",
+    ) -> dict:
+        return self._post("/v1/prompts", {
+            "project": project,
+            "prompt_name": prompt_name,
+            "content": content,
+            "model": model,
+            "temperature": temperature,
+            "commit_message": commit_message,
+            "author": author,
+        })
+
+    def list_prompt_catalog(self, project: str) -> list[dict]:
+        return self._get("/v1/prompts/catalog", {"project": project})

{cortexops-0.4.0 → cortexops-0.5.0}/cortexops/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cortexops"
-version = "0.4.0"
+version = "0.5.0"
 description = "AI agent observability & eval platform — trace, monitor, and eval-gate LangGraph, CrewAI, OpenAI Agents, PydanticAI, Agno, AutoGen, Haystack, DSPy, LlamaIndex, Smolagents, and Google ADK"
 readme = "README.md"
 license = { file = "LICENSE" }

{cortexops-0.4.0 → cortexops-0.5.0}/cortexops/tracer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import random as _random
 import time
 import uuid
 from collections.abc import Callable
@@ -90,7 +91,11 @@ class CortexTracer:
         self.api_url     = _resolve_api_url(api_url)
         # Environment: arg > env var > "development"
         self.environment = environment or os.getenv(_ENV_ENV, "development")
+        # Validate and store sample rate (0.0 = trace nothing, 1.0 = trace everything)
+        if not 0.0 <= sample_rate <= 1.0:
+            raise ValueError(f"sample_rate must be between 0.0 and 1.0, got {sample_rate}")
         self.sample_rate = sample_rate
+        self.sample_errors = True  # always trace failures regardless of sample_rate
         self.local_store = local_store
         self._traces: list[Trace] = []
         self._current_trace: Trace | None = None
@@ -129,21 +134,21 @@ class CortexTracer:
         # CrewAI
         if name == "Crew" and "crewai" in module:
             return "crewai"
-        # OpenAI Agents SDK — Agent class
-        if name == "Agent" and "agents" in module:
-            return "openai_agents"
+        # Google ADK (before OpenAI Agents — module path contains "agents")
+        if "google" in module and "adk" in module:
+            return "google_adk"
         # PydanticAI — Agent class
         if name == "Agent" and "pydantic_ai" in module:
             return "pydantic_ai"
+        # OpenAI Agents SDK — top-level `agents` package only
+        if name == "Agent" and (module == "agents" or module.startswith("agents.")):
+            return "openai_agents"
         # Agno / Phidata
         if "agno" in module or "phi" in module:
             return "agno"
         # AutoGen — ConversableAgent, AssistantAgent, UserProxyAgent
         if "autogen" in module and hasattr(agent, "initiate_chat"):
             return "autogen"
-        # Google ADK
-        if "google" in module and "adk" in module:
-            return "google_adk"
         # Smolagents (HuggingFace)
         if "smolagents" in module and hasattr(agent, "run"):
             return "smolagents"
@@ -617,10 +622,30 @@ class CortexTracer:
 
 
     def _run_traced(self, fn: Callable, input: dict, framework: str) -> Any:
-        import random
-        if self.sample_rate < 1.0 and random.random() > self.sample_rate:
+        # Head-based sampling: decide whether to trace this run.
+        # Errors are always traced (if sample_errors=True) so you never
+        # miss a failure even at low sample rates.
+        sampled = self.sample_rate >= 1.0 or _random.random() < self.sample_rate
+
+        if not sampled and not self.sample_errors:
+            # Not sampled and not capturing errors — run untraced
             return fn()
 
+        if not sampled and self.sample_errors:
+            # Not sampled, but we still want to catch errors.
+            # Run with lightweight error-only capture.
+            try:
+                return fn()
+            except Exception as exc:
+                # An error occurred — capture it even though not sampled
+                trace = Trace(project=self.project, input=input)
+                trace.status = RunStatus.FAILED
+                trace.failure_kind = FailureKind.UNKNOWN
+                trace.failure_detail = str(exc)
+                trace.metadata = {"sampled": False, "captured_reason": "error"}
+                self._traces.append(trace)
+                raise
+
         trace = Trace(project=self.project, input=input)
         self._current_trace = trace
         t0 = time.perf_counter()

{cortexops-0.4.0 → cortexops-0.5.0}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cortexops"
-version = "0.4.0"
-description = "Reliability infrastructure for AI agents — evaluation, observability, and regression testing"
+version = "0.5.0"
+description = "AI agent observability & eval platform — trace, monitor, and eval-gate LangGraph, CrewAI, OpenAI Agents, PydanticAI, Agno, AutoGen, Haystack, DSPy, LlamaIndex, Smolagents, and Google ADK"
 readme = "README.md"
 license = { file = "LICENSE" }
 authors = [
@@ -33,6 +33,7 @@ requires-python = ">=3.10"
 dependencies = [
     "pydantic>=2.0",
     "pyyaml>=6.0",
+    "httpx>=0.27",
 ]
 
 [project.optional-dependencies]

cortexops-0.5.0/tests/__init__.py

@@ -0,0 +1 @@
+# Test package for CortexOps SDK integration tests.

cortexops-0.5.0/tests/conftest.py

@@ -0,0 +1,38 @@
+"""Shared fixtures for CortexOps SDK integration tests."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.helpers import make_framework_mock, make_trace
+
+__all__ = ["make_framework_mock", "make_trace"]
+
+
+@pytest.fixture
+def project() -> str:
+    return "payments-agent"
+
+
+@pytest.fixture
+def tracer(project):
+    from cortexops import CortexTracer
+
+    return CortexTracer(project=project, sample_rate=1.0, local_store=True)
+
+
+@pytest.fixture
+def echo_agent():
+    def _agent(inp: dict) -> dict:
+        text = inp.get("input", inp.get("prompt", ""))
+        return {"output": f"Processed: {text}"}
+
+    return _agent
+
+
+@pytest.fixture
+def failing_agent():
+    def _agent(inp: dict) -> dict:
+        raise RuntimeError("agent exploded")
+
+    return _agent

cortexops-0.5.0/tests/helpers.py

@@ -0,0 +1,51 @@
+"""Shared helpers for CortexOps SDK integration tests."""
+
+from __future__ import annotations
+
+from cortexops.models import (
+    RunStatus,
+    ToolCall,
+    ToolCallStatus,
+    Trace,
+    TraceNode,
+)
+
+
+def make_trace(
+    output: dict | None = None,
+    tool_calls: list[str] | None = None,
+    latency_ms: float = 100.0,
+    *,
+    project: str = "test",
+    llm_prompt: str | None = None,
+    llm_response: str | None = None,
+    parent_nodes: list[TraceNode] | None = None,
+) -> Trace:
+    tcs = [ToolCall(name=n, status=ToolCallStatus.SUCCESS) for n in (tool_calls or [])]
+    nodes = parent_nodes or [
+        TraceNode(
+            node_id="n1",
+            node_name="agent",
+            output=output or {},
+            tool_calls=tcs,
+            latency_ms=latency_ms,
+            llm_prompt=llm_prompt,
+            llm_response=llm_response,
+        )
+    ]
+    return Trace(
+        project=project,
+        total_latency_ms=latency_ms,
+        output=output or {},
+        nodes=nodes,
+        status=RunStatus.COMPLETED,
+    )
+
+
+def make_framework_mock(class_name: str, module: str, **attrs):
+    """Build a lightweight stand-in with the given class name/module for detection."""
+    cls = type(class_name, (), {"__module__": module})
+    obj = cls()
+    for key, value in attrs.items():
+        setattr(obj, key, value)
+    return obj

cortexops-0.5.0/tests/test_agno.py

@@ -0,0 +1,7 @@
+from cortexops.tracer import CortexTracer
+from tests.helpers import make_framework_mock
+
+
+def test_agno_detected():
+    mock = make_framework_mock("Agent", "agno.agent.agent")
+    assert CortexTracer._detect_framework(mock) == "agno"

cortexops-0.5.0/tests/test_api_client.py

@@ -0,0 +1,25 @@
+"""CortexClient HTTPS upload path (mocked)."""
+
+from unittest.mock import MagicMock, patch
+
+from cortexops.client import CortexClient
+from tests.helpers import make_trace
+
+
+def test_upload_trace():
+    client = CortexClient(api_key="cxo-test", base_url="https://api.getcortexops.com")
+    trace = make_trace({"output": "ok"}, project="payments-agent")
+    with patch.object(client, "_post", return_value={"id": trace.trace_id}) as post:
+        resp = client.push_trace(trace)
+    assert resp["id"] == trace.trace_id
+    post.assert_called_once()
+    args, kwargs = post.call_args
+    assert args[0] == "/v1/traces"
+
+
+def test_list_traces():
+    client = CortexClient(api_key="cxo-test", base_url="https://api.getcortexops.com")
+    with patch.object(client, "_get", return_value=[{"trace_id": "abc"}]) as get:
+        rows = client.list_traces("payments-agent", limit=10)
+    assert rows[0]["trace_id"] == "abc"
+    get.assert_called_once()

cortexops-0.5.0/tests/test_async.py

@@ -0,0 +1,18 @@
+"""Async-friendly wrappers exist for supported frameworks."""
+
+import asyncio
+from unittest.mock import MagicMock
+
+from cortexops import CortexTracer
+from tests.helpers import make_framework_mock
+
+
+def test_langgraph_ainvoke_exists():
+    tracer = CortexTracer(project="p", sample_rate=1.0)
+    mock = make_framework_mock("CompiledStateGraph", "langgraph.graph.graph")
+    mock.invoke = MagicMock(return_value={"ok": True})
+    wrapped = tracer.wrap(mock)
+    assert hasattr(wrapped, "ainvoke")
+
+    result = asyncio.run(wrapped.ainvoke({"messages": []}))
+    assert result["ok"] is True

cortexops-0.5.0/tests/test_auth.py

@@ -0,0 +1,17 @@
+"""Credential save/load/clear."""
+
+from cortexops.auth import save_credentials, load_credentials, clear_credentials
+
+
+def test_save_load_clear(tmp_path, monkeypatch):
+    cred_dir = tmp_path / ".cortexops"
+    cred_file = cred_dir / "credentials"
+    monkeypatch.setattr("cortexops.auth._CREDENTIALS_DIR", cred_dir)
+    monkeypatch.setattr("cortexops.auth._CREDENTIALS_FILE", cred_file)
+
+    save_credentials("cxo-testkey", "payments-agent")
+    creds = load_credentials()
+    assert creds["api_key"] == "cxo-testkey"
+    assert creds["project"] == "payments-agent"
+    clear_credentials()
+    assert load_credentials() is None

cortexops-0.5.0/tests/test_autogen.py

@@ -0,0 +1,12 @@
+from unittest.mock import MagicMock
+from cortexops.tracer import CortexTracer
+from tests.helpers import make_framework_mock
+
+
+def test_autogen_detected():
+    mock = make_framework_mock(
+        "AssistantAgent",
+        "autogen.agentchat.assistant_agent",
+        initiate_chat=MagicMock(),
+    )
+    assert CortexTracer._detect_framework(mock) == "autogen"

cortexops-0.5.0/tests/test_cli.py

@@ -0,0 +1,15 @@
+"""CLI entrypoints."""
+
+from cortexops.cli import cmd_version
+from argparse import Namespace
+
+
+def test_cli_eval():
+    """Version command is wired and returns success."""
+    assert cmd_version(Namespace()) == 0
+
+
+def test_cli_main_importable():
+    from cortexops.cli import main
+
+    assert callable(main)

cortexops-0.5.0/tests/test_concurrency.py

@@ -0,0 +1,17 @@
+"""Tracer is usable across sequential calls (thread-local MVP)."""
+
+from cortexops import CortexTracer
+
+
+def test_sequential_traces_isolated():
+    tracer = CortexTracer(project="p", sample_rate=1.0)
+
+    def agent(inp: dict) -> dict:
+        return {"output": inp["input"]}
+
+    wrapped = tracer.wrap(agent)
+    wrapped({"input": "one"})
+    wrapped({"input": "two"})
+    traces = tracer.traces()
+    assert len(traces) == 2
+    assert traces[0].trace_id != traces[1].trace_id

cortexops-0.5.0/tests/test_config.py

@@ -0,0 +1,26 @@
+"""Tracer configuration."""
+
+from pathlib import Path
+
+import pytest
+
+from cortexops import CortexTracer
+
+
+def test_sample_rate_bounds():
+    with pytest.raises(ValueError):
+        CortexTracer(project="p", sample_rate=1.5)
+
+
+def test_project_from_env(monkeypatch):
+    monkeypatch.setenv("CORTEXOPS_PROJECT", "env-project")
+    t = CortexTracer(sample_rate=1.0)
+    assert t.project == "env-project"
+
+
+def test_local_only_without_key(monkeypatch, tmp_path):
+    monkeypatch.delenv("CORTEXOPS_API_KEY", raising=False)
+    monkeypatch.setattr("cortexops.tracer._CREDENTIALS_FILE", tmp_path / "missing-credentials")
+    t = CortexTracer(project="local", api_key=None)
+    assert t.is_hosted is False
+    assert isinstance(tmp_path, Path)

cortexops-0.5.0/tests/test_cost_tracking.py

@@ -0,0 +1,10 @@
+"""Cost-related fields on traces (latency proxy for MVP)."""
+
+from tests.helpers import make_trace
+
+
+def test_cost_proxy_latency():
+    t = make_trace({"output": "ok"}, latency_ms=250)
+    # MVP: cost analytics uses latency + tool count as proxies
+    cost_signal = {"latency_ms": t.total_latency_ms, "tool_calls": t.total_tool_calls()}
+    assert cost_signal["latency_ms"] == 250

cortexops-0.5.0/tests/test_crewai.py

@@ -0,0 +1,12 @@
+from unittest.mock import MagicMock
+from cortexops import CortexTracer, RunStatus
+from tests.helpers import make_framework_mock
+
+
+def test_crewai_wrap_traces():
+    tracer = CortexTracer(project="test", sample_rate=1.0)
+    mock = make_framework_mock("Crew", "crewai.crew")
+    mock.kickoff = MagicMock(return_value="done")
+    wrapped = tracer.wrap(mock)
+    assert wrapped.kickoff(inputs={"ticket": "1"}) == "done"
+    assert tracer.last_trace().status == RunStatus.COMPLETED

cortexops-0.5.0/tests/test_datadog.py

@@ -0,0 +1,8 @@
+from tests.helpers import make_trace
+
+
+def test_datadog_span_tags():
+    t = make_trace({"output": "ok"}, latency_ms=10)
+    tags = [f"project:{t.project}", f"status:{t.status.value}"]
+    assert "project:test" in tags
+    assert "status:completed" in tags

cortexops-0.5.0/tests/test_edge_cases.py

@@ -0,0 +1,21 @@
+"""Edge cases."""
+
+from cortexops import CortexTracer, RunStatus
+
+
+def test_empty_input(tracer):
+    def agent(inp: dict) -> dict:
+        return {"output": "ok"}
+
+    wrapped = tracer.wrap(agent)
+    wrapped({})
+    assert tracer.last_trace().status == RunStatus.COMPLETED
+
+
+def test_non_dict_output(tracer):
+    def agent(inp: dict):
+        return "plain-string"
+
+    wrapped = tracer.wrap(agent)
+    assert wrapped({"input": "x"}) == "plain-string"
+    assert tracer.last_trace() is not None

cortexops-0.5.0/tests/test_end_to_end.py

@@ -0,0 +1,46 @@
+"""
+End-to-end MVP path:
+
+  Agent frameworks → CortexOps SDK (tracer.wrap) → local traces
+  → metrics / eval gate → (optional) HTTPS upload via CortexClient
+"""
+
+from unittest.mock import patch
+
+from cortexops import CortexTracer, EvalSuite, CortexClient
+from cortexops.models import EvalCase, EvalDataset
+from tests.helpers import make_trace
+
+
+def test_sdk_to_cloud_path():
+    tracer = CortexTracer(project="payments-agent", sample_rate=1.0)
+
+    def agent(inp: dict) -> dict:
+        return {"output": f"hello {inp.get('input', '')}"}
+
+    wrapped = tracer.wrap(agent)
+    wrapped({"input": "world"})
+    trace = tracer.last_trace()
+    assert trace.project == "payments-agent"
+    assert "hello" in str(trace.output)
+
+    ds = EvalDataset(
+        version=1,
+        project="payments-agent",
+        cases=[EvalCase(id="c1", input="world", expected_output_contains=["hello"])],
+    )
+    summary = EvalSuite.run(dataset=ds, agent=agent, verbose=False)
+    assert summary.passed == 1
+
+    client = CortexClient(api_key="cxo-test", base_url="https://api.getcortexops.com")
+    with patch.object(client, "push_trace", return_value={"ok": True}) as push:
+        client.push_trace(trace)
+    push.assert_called_once()
+
+
+def test_dashboard_surface_fields():
+    """Fields the dashboard consumes are present on traces/evals."""
+    t = make_trace({"output": "ok"}, tool_calls=["lookup"], latency_ms=33, project="payments-agent")
+    payload = t.model_dump(mode="json")
+    for key in ("trace_id", "project", "status", "total_latency_ms", "nodes", "failure_kind"):
+        assert key in payload

cortexops-0.5.0/tests/test_eval_suite.py

@@ -0,0 +1,20 @@
+"""EvalSuite summary rendering."""
+
+from cortexops import EvalSuite
+from cortexops.models import EvalCase, EvalDataset
+
+
+def test_summary_string_renders():
+    ds = EvalDataset(
+        version=1,
+        project="test-agent",
+        cases=[EvalCase(id="c1", input="hi", expected_output_contains=["hi"])],
+    )
+
+    def agent(inp: dict) -> dict:
+        return {"output": inp.get("input", "")}
+
+    summary = EvalSuite.run(dataset=ds, agent=agent, verbose=False)
+    text = summary.summary()
+    assert "test-agent" in text
+    assert "Task completion" in text