cortexops 0.1.0__py3-none-any.whl

cortexops/tracer.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import time
+import uuid
+from contextlib import contextmanager
+from typing import Any, Callable
+
+from .models import FailureKind, RunStatus, Trace, TraceNode, ToolCall, ToolCallStatus
+
+
+class CortexTracer:
+    """Instruments AI agents with zero-refactor tracing.
+
+    Supports LangGraph StateGraph and CrewAI Crew out of the box.
+    Falls back to a generic callable wrapper for any other agent type.
+
+    Usage:
+        tracer = CortexTracer(project="payments-agent")
+        graph = tracer.wrap(your_langgraph_app)
+        result = graph.invoke({"messages": [...]})
+        trace  = tracer.last_trace()
+    """
+
+    def __init__(
+        self,
+        project: str,
+        api_key: str | None = None,
+        environment: str = "development",
+        sample_rate: float = 1.0,
+        local_store: bool = True,
+    ) -> None:
+        self.project = project
+        self.api_key = api_key
+        self.environment = environment
+        self.sample_rate = sample_rate
+        self.local_store = local_store
+        self._traces: list[Trace] = []
+        self._current_trace: Trace | None = None
+
+    def wrap(self, agent: Any) -> Any:
+        """Auto-detect agent type and return an instrumented wrapper."""
+        agent_type = type(agent).__name__
+
+        if agent_type == "CompiledStateGraph":
+            return self._wrap_langgraph(agent)
+
+        if agent_type == "Crew":
+            return self._wrap_crewai(agent)
+
+        if callable(agent) or hasattr(agent, "invoke"):
+            return self._wrap_callable(agent)
+
+        raise TypeError(
+            f"CortexTracer.wrap() does not support {agent_type}. "
+            "Pass a LangGraph CompiledStateGraph, CrewAI Crew, or any callable."
+        )
+
+    def _wrap_langgraph(self, graph: Any) -> Any:
+        tracer = self
+
+        class InstrumentedGraph:
+            def invoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
+                return tracer._run_traced(
+                    fn=lambda: graph.invoke(input, config, **kwargs),
+                    input=input,
+                    framework="langgraph",
+                )
+
+            async def ainvoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
+                import asyncio
+                return await asyncio.get_event_loop().run_in_executor(
+                    None, lambda: tracer._run_traced(
+                        fn=lambda: graph.invoke(input, config, **kwargs),
+                        input=input,
+                        framework="langgraph",
+                    )
+                )
+
+            def stream(self_, input: dict, config: dict | None = None, **kwargs):
+                return graph.stream(input, config, **kwargs)
+
+            def __getattr__(self_, name: str):
+                return getattr(graph, name)
+
+        return InstrumentedGraph()
+
+    def _wrap_crewai(self, crew: Any) -> Any:
+        tracer = self
+
+        class InstrumentedCrew:
+            def kickoff(self_, inputs: dict | None = None) -> Any:
+                return tracer._run_traced(
+                    fn=lambda: crew.kickoff(inputs=inputs),
+                    input=inputs or {},
+                    framework="crewai",
+                )
+
+            def __getattr__(self_, name: str):
+                return getattr(crew, name)
+
+        return InstrumentedCrew()
+
+    def _wrap_callable(self, fn: Any) -> Any:
+        tracer = self
+
+        if hasattr(fn, "invoke"):
+            # Object with .invoke() — wrap that method
+            original_invoke = fn.invoke
+
+            class InvokeWrapper:
+                def invoke(self_, *args, **kwargs):
+                    input_data = args[0] if args else kwargs
+                    return tracer._run_traced(
+                        fn=lambda: original_invoke(*args, **kwargs),
+                        input=input_data if isinstance(input_data, dict) else {"input": input_data},
+                        framework="generic",
+                    )
+
+                def __getattr__(self_, name: str):
+                    return getattr(fn, name)
+
+            return InvokeWrapper()
+
+        # Plain callable
+        def wrapper(*args, **kwargs):
+            input_data = {"args": list(args), "kwargs": kwargs}
+            return tracer._run_traced(fn=lambda: fn(*args, **kwargs), input=input_data, framework="generic")
+
+        return wrapper
+
+    def _run_traced(self, fn: Callable, input: dict, framework: str) -> Any:
+        trace = Trace(
+            project=self.project,
+            input=input,
+        )
+        self._current_trace = trace
+        t0 = time.perf_counter()
+
+        try:
+            result = fn()
+            trace.total_latency_ms = (time.perf_counter() - t0) * 1000
+            trace.status = RunStatus.COMPLETED
+            trace.output = result if isinstance(result, dict) else {"result": str(result)}
+        except Exception as exc:
+            trace.total_latency_ms = (time.perf_counter() - t0) * 1000
+            trace.status = RunStatus.FAILED
+            trace.failure_kind = FailureKind.UNKNOWN
+            trace.failure_detail = str(exc)
+            raise
+        finally:
+            self._traces.append(trace)
+            if self.api_key:
+                self._flush_trace(trace)
+
+        return result
+
+    @contextmanager
+    def trace_node(self, node_name: str):
+        """Context manager to manually instrument a single node."""
+        node = TraceNode(node_id=str(uuid.uuid4()), node_name=node_name)
+        t0 = time.perf_counter()
+        try:
+            yield node
+        finally:
+            node.latency_ms = (time.perf_counter() - t0) * 1000
+            if self._current_trace:
+                self._current_trace.nodes.append(node)
+
+    def record_tool_call(
+        self,
+        name: str,
+        args: dict | None = None,
+        result: Any = None,
+        error: str | None = None,
+        latency_ms: float = 0.0,
+    ) -> ToolCall:
+        """Manually record a tool call onto the current active trace."""
+        tc = ToolCall(
+            name=name,
+            args=args or {},
+            result=result,
+            status=ToolCallStatus.ERROR if error else ToolCallStatus.SUCCESS,
+            latency_ms=latency_ms,
+            error=error,
+        )
+        if self._current_trace and self._current_trace.nodes:
+            self._current_trace.nodes[-1].tool_calls.append(tc)
+        return tc
+
+    def last_trace(self) -> Trace | None:
+        return self._traces[-1] if self._traces else None
+
+    def traces(self) -> list[Trace]:
+        return list(self._traces)
+
+    def clear(self) -> None:
+        self._traces.clear()
+        self._current_trace = None
+
+    def _flush_trace(self, trace: Trace) -> None:
+        try:
+            import httpx
+            httpx.post(
+                "https://api.cortexops.ai/v1/traces",
+                json=trace.model_dump(mode="json"),
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                timeout=2.0,
+            )
+        except Exception:
+            pass  # non-blocking — tracing never breaks the agent

cortexops-0.1.0.dist-info/METADATA

@@ -0,0 +1,169 @@
+Metadata-Version: 2.4
+Name: cortexops
+Version: 0.1.0
+Summary: Reliability infrastructure for AI agents — evaluation, observability, and regression testing
+Project-URL: Homepage, https://cortexops.ai
+Project-URL: Repository, https://github.com/ashishodu2023/cortexops
+Project-URL: Documentation, https://docs.cortexops.ai
+Project-URL: Bug Tracker, https://github.com/ashishodu2023/cortexops/issues
+Project-URL: Changelog, https://github.com/ashishodu2023/cortexops/releases
+Author-email: Ashish <ashishodu2023@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 CortexOps Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,ai,autogen,crewai,evaluation,langgraph,llm,observability,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: setuptools>=82.0.1
+Provides-Extra: all
+Requires-Dist: httpx>=0.27; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: http
+Requires-Dist: httpx>=0.27; extra == 'http'
+Provides-Extra: llm
+Requires-Dist: httpx>=0.27; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/ashishodu2023/cortexops/blob/main/LICENSE)
+
+---
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Install
+
+```bash
+pip install cortexops
+
+# With HTTP client (for pushing traces to hosted API):
+pip install cortexops[http]
+
+# With LLM judge support:
+pip install cortexops[llm]
+```
+
+---
+
+## Quickstart
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+print(results.summary())
+```
+
+---
+
+## Golden dataset (YAML)
+
+```yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: open_ended_explanation_01
+    input: "Why was my refund rejected?"
+    judge: llm
+    judge_criteria: >
+      The response must explain the rejection reason clearly,
+      be empathetic, and offer a concrete next step. No jargon.
+```
+
+---
+
+## CI gate
+
+```bash
+cortexops eval run \
+  --dataset golden_v1.yaml \
+  --fail-on "task_completion < 0.90"
+```
+
+Exits non-zero if the threshold is not met — blocks the PR.
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Non-empty, non-error output with expected content |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Fabrication signals in output |
+| `llm_judge` | GPT-4o scores against natural-language criteria |
+
+---
+
+## Links
+
+- **Docs**: [docs.cortexops.ai](https://docs.cortexops.ai)
+- **Repo**: [github.com/ashishodu2023/cortexops](https://github.com/ashishodu2023/cortexops)
+- **Issues**: [GitHub Issues](https://github.com/ashishodu2023/cortexops/issues)

cortexops-0.1.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+cortexops/LICENSE,sha256=cXemgb-9EkUWB7EQ1riVRZn2gVQ1JcG7U34LpkUjaZA,1079
+cortexops/README.md,sha256=ydnSnOOqE2BbXgpFiusbaNQORc69ANkhQoeoJZAmHhc,2720
+cortexops/__init__.py,sha256=6IBgalIuUUlXz3TlliO3ERUpi-FJQwFE71CY1HY1C5s,1168
+cortexops/cli.py,sha256=oZKt6Xow6srU3Xm5GJ0-5OsfQAyHphPxzFNJIqHLU0s,7281
+cortexops/client.py,sha256=AE9hhcdlP2D-_QwQN0Qj4572WNqVykYTB0JOi7917R0,2754
+cortexops/eval.py,sha256=BkVoYLDzx15ZhJ0V-whUROYqBvSUgM_3l10L7yQu5yA,7248
+cortexops/judge.py,sha256=ILJNaTySkfqT-dAnafmr48bb9N9YiDXoNo_JmZLQUBM,5386
+cortexops/metrics.py,sha256=BMK8I0ceabpo0yZvP5lVvL9lCPBCZ8yNptidYeXwIK8,6545
+cortexops/models.py,sha256=9mx2ZUAJJyzSQXmTsVqJfLRLorRyuQ_MIhmnHrXYABE,4441
+cortexops/pyproject.toml,sha256=rDbZVYog_dZ7gg7mR_RVNgCMNulvcn70nK7AIIArbl8,2164
+cortexops/tracer.py,sha256=ySoDkUZwkSzWUnk9cs9puf_Z3Wz4HdvGhxbOSbjIlcw,7108
+cortexops/cortexops/__init__.py,sha256=6IBgalIuUUlXz3TlliO3ERUpi-FJQwFE71CY1HY1C5s,1168
+cortexops/cortexops/cli.py,sha256=oZKt6Xow6srU3Xm5GJ0-5OsfQAyHphPxzFNJIqHLU0s,7281
+cortexops/cortexops/client.py,sha256=AE9hhcdlP2D-_QwQN0Qj4572WNqVykYTB0JOi7917R0,2754
+cortexops/cortexops/eval.py,sha256=BkVoYLDzx15ZhJ0V-whUROYqBvSUgM_3l10L7yQu5yA,7248
+cortexops/cortexops/judge.py,sha256=ILJNaTySkfqT-dAnafmr48bb9N9YiDXoNo_JmZLQUBM,5386
+cortexops/cortexops/metrics.py,sha256=BMK8I0ceabpo0yZvP5lVvL9lCPBCZ8yNptidYeXwIK8,6545
+cortexops/cortexops/models.py,sha256=9mx2ZUAJJyzSQXmTsVqJfLRLorRyuQ_MIhmnHrXYABE,4441
+cortexops/cortexops/tracer.py,sha256=ySoDkUZwkSzWUnk9cs9puf_Z3Wz4HdvGhxbOSbjIlcw,7108
+cortexops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cortexops/tests/test_cortexops.py,sha256=SVoeR3ZfCca4J0Kg4-HM4_MPmoTGSElVcPC-OQqdYRM,7967
+cortexops/tests/test_enhancements.py,sha256=gqxrwuF9EuDxunLlqNFmwSCoq9gm0BrYN81ZgM3WBLc,8238
+cortexops-0.1.0.dist-info/METADATA,sha256=UCATLqZfsI3Y9RMj99ZjMmS2MvJqUNVswfO0byXpgAo,5783
+cortexops-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+cortexops-0.1.0.dist-info/entry_points.txt,sha256=wBc4X1RuV2sUDyF5TLPmsRrJQ3GrbewjPeC0K0C6r0k,49
+cortexops-0.1.0.dist-info/licenses/LICENSE,sha256=cXemgb-9EkUWB7EQ1riVRZn2gVQ1JcG7U34LpkUjaZA,1079
+cortexops-0.1.0.dist-info/RECORD,,

cortexops-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

cortexops-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cortexops = cortexops.cli:main

cortexops-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 CortexOps Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.