cortexops 0.1.0__tar.gz

cortexops-0.1.0/.gitignore

@@ -0,0 +1,34 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+.venv/
+venv/
+.env
+*.egg-info/
+dist/
+build/
+PKG-INFO
+*.whl
+*.tar.gz
+
+# Test / lint caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# Package managers
+uv.lock
+.python-version
+
+# Database
+*.db
+*.sqlite
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db

cortexops-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 CortexOps Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cortexops-0.1.0/PKG-INFO

@@ -0,0 +1,169 @@
+Metadata-Version: 2.4
+Name: cortexops
+Version: 0.1.0
+Summary: Reliability infrastructure for AI agents — evaluation, observability, and regression testing
+Project-URL: Homepage, https://cortexops.ai
+Project-URL: Repository, https://github.com/ashishodu2023/cortexops
+Project-URL: Documentation, https://docs.cortexops.ai
+Project-URL: Bug Tracker, https://github.com/ashishodu2023/cortexops/issues
+Project-URL: Changelog, https://github.com/ashishodu2023/cortexops/releases
+Author-email: Ashish <ashishodu2023@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 CortexOps Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,ai,autogen,crewai,evaluation,langgraph,llm,observability,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: setuptools>=82.0.1
+Provides-Extra: all
+Requires-Dist: httpx>=0.27; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: http
+Requires-Dist: httpx>=0.27; extra == 'http'
+Provides-Extra: llm
+Requires-Dist: httpx>=0.27; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/ashishodu2023/cortexops/blob/main/LICENSE)
+
+---
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Install
+
+```bash
+pip install cortexops
+
+# With HTTP client (for pushing traces to hosted API):
+pip install cortexops[http]
+
+# With LLM judge support:
+pip install cortexops[llm]
+```
+
+---
+
+## Quickstart
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+print(results.summary())
+```
+
+---
+
+## Golden dataset (YAML)
+
+```yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: open_ended_explanation_01
+    input: "Why was my refund rejected?"
+    judge: llm
+    judge_criteria: >
+      The response must explain the rejection reason clearly,
+      be empathetic, and offer a concrete next step. No jargon.
+```
+
+---
+
+## CI gate
+
+```bash
+cortexops eval run \
+  --dataset golden_v1.yaml \
+  --fail-on "task_completion < 0.90"
+```
+
+Exits non-zero if the threshold is not met — blocks the PR.
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Non-empty, non-error output with expected content |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Fabrication signals in output |
+| `llm_judge` | GPT-4o scores against natural-language criteria |
+
+---
+
+## Links
+
+- **Docs**: [docs.cortexops.ai](https://docs.cortexops.ai)
+- **Repo**: [github.com/ashishodu2023/cortexops](https://github.com/ashishodu2023/cortexops)
+- **Issues**: [GitHub Issues](https://github.com/ashishodu2023/cortexops/issues)

cortexops-0.1.0/README.md

@@ -0,0 +1,106 @@
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/ashishodu2023/cortexops/blob/main/LICENSE)
+
+---
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Install
+
+```bash
+pip install cortexops
+
+# With HTTP client (for pushing traces to hosted API):
+pip install cortexops[http]
+
+# With LLM judge support:
+pip install cortexops[llm]
+```
+
+---
+
+## Quickstart
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+print(results.summary())
+```
+
+---
+
+## Golden dataset (YAML)
+
+```yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: open_ended_explanation_01
+    input: "Why was my refund rejected?"
+    judge: llm
+    judge_criteria: >
+      The response must explain the rejection reason clearly,
+      be empathetic, and offer a concrete next step. No jargon.
+```
+
+---
+
+## CI gate
+
+```bash
+cortexops eval run \
+  --dataset golden_v1.yaml \
+  --fail-on "task_completion < 0.90"
+```
+
+Exits non-zero if the threshold is not met — blocks the PR.
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Non-empty, non-error output with expected content |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Fabrication signals in output |
+| `llm_judge` | GPT-4o scores against natural-language criteria |
+
+---
+
+## Links
+
+- **Docs**: [docs.cortexops.ai](https://docs.cortexops.ai)
+- **Repo**: [github.com/ashishodu2023/cortexops](https://github.com/ashishodu2023/cortexops)
+- **Issues**: [GitHub Issues](https://github.com/ashishodu2023/cortexops/issues)

cortexops-0.1.0/cortexops/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 CortexOps Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cortexops-0.1.0/cortexops/README.md

@@ -0,0 +1,106 @@
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/ashishodu2023/cortexops/blob/main/LICENSE)
+
+---
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Install
+
+```bash
+pip install cortexops
+
+# With HTTP client (for pushing traces to hosted API):
+pip install cortexops[http]
+
+# With LLM judge support:
+pip install cortexops[llm]
+```
+
+---
+
+## Quickstart
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+print(results.summary())
+```
+
+---
+
+## Golden dataset (YAML)
+
+```yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: open_ended_explanation_01
+    input: "Why was my refund rejected?"
+    judge: llm
+    judge_criteria: >
+      The response must explain the rejection reason clearly,
+      be empathetic, and offer a concrete next step. No jargon.
+```
+
+---
+
+## CI gate
+
+```bash
+cortexops eval run \
+  --dataset golden_v1.yaml \
+  --fail-on "task_completion < 0.90"
+```
+
+Exits non-zero if the threshold is not met — blocks the PR.
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Non-empty, non-error output with expected content |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Fabrication signals in output |
+| `llm_judge` | GPT-4o scores against natural-language criteria |
+
+---
+
+## Links
+
+- **Docs**: [docs.cortexops.ai](https://docs.cortexops.ai)
+- **Repo**: [github.com/ashishodu2023/cortexops](https://github.com/ashishodu2023/cortexops)
+- **Issues**: [GitHub Issues](https://github.com/ashishodu2023/cortexops/issues)

cortexops-0.1.0/cortexops/__init__.py

@@ -0,0 +1,58 @@
+"""CortexOps — Reliability infrastructure for AI agents.
+
+Quickstart:
+    from cortexops import CortexTracer, EvalSuite
+
+    tracer = CortexTracer(project="my-agent")
+    graph  = tracer.wrap(your_langgraph_app)
+
+    results = EvalSuite.run(dataset="golden_v1.yaml", agent=graph)
+    print(results.summary())
+"""
+
+from .client import CortexClient
+from .eval import EvalSuite, EvalThresholdError
+from .judge import LLMJudgeMetric
+from .metrics import (
+    HallucinationMetric,
+    LatencyMetric,
+    Metric,
+    TaskCompletionMetric,
+    ToolAccuracyMetric,
+)
+from .models import (
+    CaseResult,
+    EvalCase,
+    EvalDataset,
+    EvalSummary,
+    FailureKind,
+    RunStatus,
+    Trace,
+    TraceNode,
+    ToolCall,
+)
+from .tracer import CortexTracer
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CortexTracer",
+    "EvalSuite",
+    "EvalThresholdError",
+    "CortexClient",
+    "Metric",
+    "TaskCompletionMetric",
+    "ToolAccuracyMetric",
+    "LatencyMetric",
+    "HallucinationMetric",
+    "LLMJudgeMetric",
+    "Trace",
+    "TraceNode",
+    "ToolCall",
+    "EvalCase",
+    "EvalDataset",
+    "EvalSummary",
+    "CaseResult",
+    "FailureKind",
+    "RunStatus",
+]