cortexops 0.3.0__tar.gz → 0.5.0__tar.gz

{cortexops-0.3.0 → cortexops-0.5.0}/.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *.pyo
+.venv-e2e/
 .venv/
 venv/
 .env

cortexops-0.5.0/PKG-INFO

@@ -0,0 +1,306 @@
+Metadata-Version: 2.4
+Name: cortexops
+Version: 0.5.0
+Summary: AI agent observability & eval platform — trace, monitor, and eval-gate LangGraph, CrewAI, OpenAI Agents, PydanticAI, Agno, AutoGen, Haystack, DSPy, LlamaIndex, Smolagents, and Google ADK
+Project-URL: Homepage, https://getcortexops.com
+Project-URL: Repository, https://github.com/ashishodu2023/cortexops
+Project-URL: Documentation, https://docs.getcortexops.com
+Project-URL: Bug Tracker, https://github.com/ashishodu2023/cortexops/issues
+Project-URL: Changelog, https://github.com/ashishodu2023/cortexops/releases
+Author-email: Ashish <ashish@getcortexops.com>
+License: MIT License
+        
+        Copyright (c) 2025 CortexOps Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,ai,autogen,crewai,evaluation,langgraph,llm,observability,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: all
+Requires-Dist: httpx>=0.27; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: http
+Requires-Dist: httpx>=0.27; extra == 'http'
+Provides-Extra: llm
+Requires-Dist: httpx>=0.27; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+---
+
+## What's New in v0.4.0
+
+### LLM-as-judge evaluation
+```python
+from cortexops.judge import LLMJudge
+
+judge = LLMJudge(api_key="sk-...")
+result = judge.evaluate(
+    case_id="case-001",
+    input="Process refund for order #4821",
+    output="Refund of $49.99 approved and processed.",
+    rubric="task_completion",
+)
+print(result.score, result.passed, result.reasoning)
+```
+
+### Golden dataset API
+```python
+from cortexops.dataset import GoldenDataset
+
+ds = GoldenDataset(name="refund-agent-v1")
+ds.add(input="Refund order #4821", expected="refund_approved")
+ds.add(input="Cancel subscription", expected="subscription_cancelled")
+ds.save("datasets/refund_agent.yaml")
+
+results = ds.run(agent=your_agent, fail_on="task_completion < 0.90")
+```
+
+### CI/CD eval gate
+```bash
+cortexops eval run \
+  --dataset datasets/refund_agent.yaml \
+  --judge \
+  --fail-on "task_completion < 0.90"
+# Exit code 1 if regression detected — drop into GitHub Actions
+```
+
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Quickstart
+
+```bash
+pip install cortexops  # v0.4.0
+```
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+
+print(results.summary())
+# CortexOps eval — payments-agent
+#   Cases           : 9  (7 passed, 2 failed)
+#   Task completion : 91.4%
+#   Tool accuracy   : 97.0/100
+#   Latency p50/p95 : 42ms / 187ms
+#   Failed cases:
+#     - escalation_router: tool_call_mismatch (score 41)
+```
+
+---
+
+## Golden dataset format
+
+Define test cases in YAML. Run them locally or in CI.
+
+```yaml
+# golden_v1.yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: dispute_escalation_01
+    input: "I was charged twice — this is unauthorized"
+    expected_tool_calls: [classify_dispute, route_escalation]
+    expected_output_contains: ["escalated"]
+    max_latency_ms: 5000
+```
+
+---
+
+## CI eval gate
+
+Add to `.github/workflows/eval.yml`:
+
+```yaml
+- name: CortexOps eval gate
+  run: |
+    python examples/langgraph_payments/run_eval.py \
+      --dataset golden_v1.yaml \
+      --fail-on "task_completion < 0.90"
+```
+
+If the eval drops below threshold, the job exits non-zero and the PR is blocked.
+
+---
+
+## Repo structure
+
+```
+cortexops/
+├── sdk/                        # pip install cortexops  # v0.4.0
+│   ├── cortexops/
+│   │   ├── tracer.py           # CortexTracer — wraps LangGraph / CrewAI
+│   │   ├── eval.py             # EvalSuite — golden dataset runner
+│   │   ├── metrics.py          # task_completion, tool_accuracy, latency, hallucination
+│   │   ├── models.py           # Pydantic data models
+│   │   └── client.py           # HTTP client for hosted API
+│   └── tests/
+├── backend/                    # FastAPI + Celery + SQLite/Postgres
+│   ├── app/
+│   │   ├── main.py
+│   │   ├── routers/            # /v1/evals, /v1/traces
+│   │   ├── models/             # DB records + API schemas
+│   │   └── worker/             # Celery async eval tasks
+│   └── Dockerfile
+├── frontend/                   # React + TypeScript dashboard
+├── examples/
+│   └── langgraph_payments/     # Full runnable demo
+│       ├── agent.py
+│       ├── golden_v1.yaml
+│       └── run_eval.py
+└── docker-compose.yml
+```
+
+---
+
+## Run the full stack locally
+
+```bash
+git clone https://github.com/ashishodu2023/cortexops
+cd cortexops
+
+# Start API + worker + Redis
+docker compose up --build
+
+# In another terminal — run the demo eval
+cd examples/langgraph_payments
+pip install -e ../../sdk/
+python run_eval.py
+
+# API docs at http://localhost:8000/docs
+# Dashboard at http://localhost:3000
+```
+
+---
+
+## Supported frameworks
+
+| Framework | Status |
+|---|---|
+| LangGraph | Stable |
+| CrewAI | Stable |
+| AutoGen | Beta |
+| LlamaIndex agents | Coming soon |
+| Custom callables | Supported via `CortexTracer.wrap()` |
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Agent produced a valid, non-error output |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Detects fabrication signals in output |
+
+Add custom metrics by subclassing `cortexops.Metric`.
+
+---
+
+## Contributing
+
+```bash
+git clone https://github.com/ashishodu2023/cortexops
+cd cortexops/sdk
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Issues labeled `good first issue` are a great place to start.
+
+---
+
+## Citation
+
+```bibtex
+@software{cortexops2025,
+  author  = {Ashish, et al.},
+  title   = {CortexOps: Reliability Infrastructure for AI Agents},
+  year    = {2025},
+  url     = {https://github.com/ashishodu2023/cortexops},
+}
+```
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+---
+
+<p align="center">
+  <a href="https://cortexops.ai">cortexops.ai</a> ·
+  <a href="https://github.com/ashishodu2023/cortexops/issues">Issues</a> ·
+  <a href="https://github.com/ashishodu2023/cortexops/discussions">Discussions</a>
+</p>

cortexops-0.5.0/README.md

@@ -0,0 +1,243 @@
+# CortexOps
+
+**Reliability infrastructure for AI agents.**  
+Evaluate · Observe · Operate — for LangGraph, CrewAI, and AutoGen.
+
+[![PyPI version](https://img.shields.io/pypi/v/cortexops.svg)](https://pypi.org/project/cortexops/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml/badge.svg)](https://github.com/ashishodu2023/cortexops/actions/workflows/eval.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+---
+
+## What's New in v0.4.0
+
+### LLM-as-judge evaluation
+```python
+from cortexops.judge import LLMJudge
+
+judge = LLMJudge(api_key="sk-...")
+result = judge.evaluate(
+    case_id="case-001",
+    input="Process refund for order #4821",
+    output="Refund of $49.99 approved and processed.",
+    rubric="task_completion",
+)
+print(result.score, result.passed, result.reasoning)
+```
+
+### Golden dataset API
+```python
+from cortexops.dataset import GoldenDataset
+
+ds = GoldenDataset(name="refund-agent-v1")
+ds.add(input="Refund order #4821", expected="refund_approved")
+ds.add(input="Cancel subscription", expected="subscription_cancelled")
+ds.save("datasets/refund_agent.yaml")
+
+results = ds.run(agent=your_agent, fail_on="task_completion < 0.90")
+```
+
+### CI/CD eval gate
+```bash
+cortexops eval run \
+  --dataset datasets/refund_agent.yaml \
+  --judge \
+  --fail-on "task_completion < 0.90"
+# Exit code 1 if regression detected — drop into GitHub Actions
+```
+
+
+## The problem
+
+You deployed an agent. You have no idea if it regressed overnight.
+
+No standard eval format. No failure traces. No CI gate before the next prompt change ships.  
+CortexOps fixes that.
+
+---
+
+## Quickstart
+
+```bash
+pip install cortexops  # v0.4.0
+```
+
+```python
+from cortexops import CortexTracer, EvalSuite
+
+# Wrap your LangGraph app — zero refactor required
+tracer = CortexTracer(project="payments-agent")
+graph  = tracer.wrap(your_langgraph_app)
+
+# Run evaluations against a golden dataset
+results = EvalSuite.run(
+    dataset="golden_v1.yaml",
+    agent=graph,
+)
+
+print(results.summary())
+# CortexOps eval — payments-agent
+#   Cases           : 9  (7 passed, 2 failed)
+#   Task completion : 91.4%
+#   Tool accuracy   : 97.0/100
+#   Latency p50/p95 : 42ms / 187ms
+#   Failed cases:
+#     - escalation_router: tool_call_mismatch (score 41)
+```
+
+---
+
+## Golden dataset format
+
+Define test cases in YAML. Run them locally or in CI.
+
+```yaml
+# golden_v1.yaml
+version: 1
+project: payments-agent
+
+cases:
+  - id: refund_lookup_01
+    input: "What is the status of refund REF-8821?"
+    expected_tool_calls: [lookup_refund]
+    expected_output_contains: ["approved", "REF-8821"]
+    max_latency_ms: 3000
+
+  - id: dispute_escalation_01
+    input: "I was charged twice — this is unauthorized"
+    expected_tool_calls: [classify_dispute, route_escalation]
+    expected_output_contains: ["escalated"]
+    max_latency_ms: 5000
+```
+
+---
+
+## CI eval gate
+
+Add to `.github/workflows/eval.yml`:
+
+```yaml
+- name: CortexOps eval gate
+  run: |
+    python examples/langgraph_payments/run_eval.py \
+      --dataset golden_v1.yaml \
+      --fail-on "task_completion < 0.90"
+```
+
+If the eval drops below threshold, the job exits non-zero and the PR is blocked.
+
+---
+
+## Repo structure
+
+```
+cortexops/
+├── sdk/                        # pip install cortexops  # v0.4.0
+│   ├── cortexops/
+│   │   ├── tracer.py           # CortexTracer — wraps LangGraph / CrewAI
+│   │   ├── eval.py             # EvalSuite — golden dataset runner
+│   │   ├── metrics.py          # task_completion, tool_accuracy, latency, hallucination
+│   │   ├── models.py           # Pydantic data models
+│   │   └── client.py           # HTTP client for hosted API
+│   └── tests/
+├── backend/                    # FastAPI + Celery + SQLite/Postgres
+│   ├── app/
+│   │   ├── main.py
+│   │   ├── routers/            # /v1/evals, /v1/traces
+│   │   ├── models/             # DB records + API schemas
+│   │   └── worker/             # Celery async eval tasks
+│   └── Dockerfile
+├── frontend/                   # React + TypeScript dashboard
+├── examples/
+│   └── langgraph_payments/     # Full runnable demo
+│       ├── agent.py
+│       ├── golden_v1.yaml
+│       └── run_eval.py
+└── docker-compose.yml
+```
+
+---
+
+## Run the full stack locally
+
+```bash
+git clone https://github.com/ashishodu2023/cortexops
+cd cortexops
+
+# Start API + worker + Redis
+docker compose up --build
+
+# In another terminal — run the demo eval
+cd examples/langgraph_payments
+pip install -e ../../sdk/
+python run_eval.py
+
+# API docs at http://localhost:8000/docs
+# Dashboard at http://localhost:3000
+```
+
+---
+
+## Supported frameworks
+
+| Framework | Status |
+|---|---|
+| LangGraph | Stable |
+| CrewAI | Stable |
+| AutoGen | Beta |
+| LlamaIndex agents | Coming soon |
+| Custom callables | Supported via `CortexTracer.wrap()` |
+
+---
+
+## Built-in metrics
+
+| Metric | What it checks |
+|---|---|
+| `task_completion` | Agent produced a valid, non-error output |
+| `tool_accuracy` | Expected tool calls were actually made |
+| `latency` | Response within `max_latency_ms` budget |
+| `hallucination` | Detects fabrication signals in output |
+
+Add custom metrics by subclassing `cortexops.Metric`.
+
+---
+
+## Contributing
+
+```bash
+git clone https://github.com/ashishodu2023/cortexops
+cd cortexops/sdk
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Issues labeled `good first issue` are a great place to start.
+
+---
+
+## Citation
+
+```bibtex
+@software{cortexops2025,
+  author  = {Ashish, et al.},
+  title   = {CortexOps: Reliability Infrastructure for AI Agents},
+  year    = {2025},
+  url     = {https://github.com/ashishodu2023/cortexops},
+}
+```
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+---
+
+<p align="center">
+  <a href="https://cortexops.ai">cortexops.ai</a> ·
+  <a href="https://github.com/ashishodu2023/cortexops/issues">Issues</a> ·
+  <a href="https://github.com/ashishodu2023/cortexops/discussions">Discussions</a>
+</p>

{cortexops-0.3.0 → cortexops-0.5.0}/cortexops/__init__.py

@@ -34,7 +34,7 @@ from .models import (
 )
 from .tracer import CortexTracer
 
-__version__ = "0.3.0"
+__version__ = "0.5.0"
 
 __all__ = [
     "CortexTracer",

{cortexops-0.3.0 → cortexops-0.5.0}/cortexops/cli.py

@@ -127,6 +127,88 @@ def cmd_version(_: argparse.Namespace) -> int:
     return 0
 
 
+
+
+def cmd_dataset_create(args: argparse.Namespace) -> int:
+    """cortexops dataset create --name my-dataset --output dataset.yaml"""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cortexops.dataset import GoldenDataset
+
+    ds = GoldenDataset(name=args.name, description=args.description or "")
+    ds.save(args.output)
+    print(f"Created dataset: {args.output}")
+    print("  Add cases by editing the YAML file or using ds.add() in Python.")
+    return 0
+
+
+def cmd_eval_judge(args: argparse.Namespace) -> int:
+    """cortexops eval judge --input <str> --output <str> --rubric task_completion"""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cortexops.judge import RUBRICS, LLMJudge
+
+    api_key = args.api_key or os.getenv("OPENAI_API_KEY", "")
+    if not api_key:
+        print("Error: OPENAI_API_KEY not set. Pass --api-key or set the env var.", file=sys.stderr)
+        return 1
+
+    judge = LLMJudge(api_key=api_key, model=args.model or "gpt-4o-mini")
+    result = judge.evaluate(
+        case_id="cli-eval",
+        input=args.input,
+        output=args.output,
+        rubric=args.rubric or "task_completion",
+        expected=args.expected,
+    )
+
+    icon = "✓ PASS" if result.passed else "✗ FAIL"
+    print(f"\nLLM Judge Result: {icon}")
+    rubric_obj   = RUBRICS.get(args.rubric or 'task_completion')
+    threshold    = rubric_obj.pass_threshold if rubric_obj else 0.70
+    print(f"  Score:     {result.score:.3f} (threshold: {threshold:.2f})")
+    print(f"  Model:     {result.model}  ({result.latency_ms}ms)")
+    print(f"  Reasoning: {result.reasoning}")
+    if args.verbose:
+        print(f"  Criteria:  {result.criteria_scores}")
+
+    return 0 if result.passed else 1
+
+
+def cmd_eval_run_with_judge(args: argparse.Namespace) -> int:
+    """cortexops eval run --dataset d.yaml --judge --fail-on task_completion<0.90"""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cortexops.dataset import GoldenDataset
+
+    ds = GoldenDataset.load(args.dataset)
+    print("CortexOps eval gate")
+    print(f"  dataset : {args.dataset} ({len(ds)} cases)")
+    print(f"  project : {args.project or ds.name}")
+    if args.fail_on:
+        print(f"  fail-on : {args.fail_on}")
+    if args.judge:
+        print(f"  judge   : LLM-as-judge ({args.model or 'gpt-4o-mini'})")
+    print()
+
+    def passthrough_agent(inp):
+        return {"output": f"[no agent] input: {inp}"}
+
+    try:
+        agent = _load_agent(args.agent) if getattr(args, "agent", None) else passthrough_agent
+        result = ds.run(
+            agent=agent,
+            fail_on=args.fail_on,
+            verbose=True,
+            use_judge=getattr(args, "judge", False),
+            judge_rubric=getattr(args, "rubric", "task_completion"),
+            judge_api_key=os.getenv("OPENAI_API_KEY"),
+        )
+        result.print_report()
+        return 0 if result.passed() else 1
+    except Exception as e:
+        print(f"\nEval failed: {e}", file=sys.stderr)
+        return 1
+
+
+
 def _load_agent(agent_path: str):
     """Load an agent from a dotted path like 'mymodule:my_agent'."""
     if ":" not in agent_path: