agenttest-py 0.1.0__tar.gz

agenttest_py-0.1.0/.github/workflows/agenttest.yml

@@ -0,0 +1,19 @@
+name: Agent Evals
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install -e .
+      - run: agenttest run
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

agenttest_py-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Build
+        run: hatch build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

agenttest_py-0.1.0/.gitignore

@@ -0,0 +1,44 @@
+# Agenttest
+.agenttest_cache/
+.agenttest_results.json
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.coverage
+htmlcov/
+.pytest_cache/
+.tox/

agenttest_py-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,38 @@
+# Contributing to agenttest
+
+Thank you for considering contributing to agenttest! This document provides guidelines for contributing.
+
+## Development Setup
+
+```bash
+git clone https://github.com/your-org/agenttest.git
+cd agenttest
+pip install -e ".[dev]"
+```
+
+## Running Tests
+
+```bash
+# Unit tests (pytest)
+pytest tests/
+
+# Agent evals (requires ANTHROPIC_API_KEY)
+agenttest run
+```
+
+## Code Style
+
+- Use type hints on all public functions
+- Add docstrings to public APIs
+- Follow existing patterns in the codebase
+
+## Pull Request Process
+
+1. Fork the repo and create a branch
+2. Make your changes with tests
+3. Ensure `pytest tests/` and `agenttest run` pass
+4. Submit a PR with a clear description
+
+## Questions?
+
+Open an issue for discussion.

agenttest_py-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ShashStudios
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agenttest_py-0.1.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: agenttest-py
+Version: 0.1.0
+Summary: The pytest of AI agents. Eval-driven testing for LLM applications.
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,anthropic,evals,llm,pytest,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.9
+Requires-Dist: anthropic>=0.20.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: toml>=0.10.0
+Description-Content-Type: text/markdown
+
+# agenttest
+
+**The pytest of AI agents.** Catch regressions before they reach prod.
+
+[![PyPI version](https://img.shields.io/pypi/v/agenttest-py.svg)](https://pypi.org/project/agenttest-py/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ShashStudios/agenttest/actions/workflows/agenttest.yml/badge.svg)](https://github.com/ShashStudios/agenttest/actions)
+
+---
+
+## You ship an agent. You change a prompt. Did it get better or worse?
+
+**You have no idea.** No test suite. No CI. No diff. Just deploy and hope.
+
+Every team shipping AI agents hits the same wall: your "eval" is manually pasting examples into a playground. One prompt tweak could break everything—or fix everything—and you won't know until a user complains.
+
+---
+
+## 30-Second Quickstart
+
+```bash
+pip install agenttest-py
+export ANTHROPIC_API_KEY=your_key
+agenttest init
+agenttest run
+```
+
+**Or from scratch:**
+
+```python
+# agent_test_example.py
+from agenttest import eval, judge
+
+def my_agent(query: str) -> str:
+    return "Your agent's response"  # Replace with real agent
+
+@eval
+def test_customer_support():
+    response = my_agent("I want a refund")
+    assert judge.tone(response) == "empathetic"
+    assert judge.no_hallucination(response)
+```
+
+```bash
+agenttest run
+```
+
+---
+
+## Features
+
+- **Code-first** — Tests are just Python. No YAML. No config hell.
+- **LLM-as-judge** — 9 built-in scorers: tone, hallucination, relevance, toxicity, faithfulness, conciseness, custom criteria, A/B compare.
+- **Local & CI** — Runs anywhere. Add 4 lines to GitHub Actions. No account. No dashboard.
+- **`agenttest diff`** — Side-by-side view of how your agent's responses changed between two runs. The git diff for agent behavior.
+- **Caching** — Judge results cached in `.agenttest_cache/` to avoid redundant API calls.
+- **Parallel** — `--workers 4` for faster runs.
+
+---
+
+## agenttest diff — The Git Diff for Agent Behavior
+
+See exactly how your agent's responses changed between two runs:
+
+```bash
+agenttest run --tag v1    # Before your prompt change
+agenttest run --tag v2    # After your prompt change
+agenttest diff v1 v2
+```
+
+```
+test_customer_support_refund:
+  BEFORE: "I cannot help with refunds"           pass
+  AFTER:  "I'd be happy to process that for you" pass
+  DELTA:  ✓ improved
+
+test_helpful_tone:
+  BEFORE: "Our policy states no returns"         fail
+  AFTER:  "I'm sorry to hear that. Let me help"   pass
+  DELTA:  +1 ↑
+```
+
+---
+
+## Judge Functions
+
+| Function | Returns | Description |
+|----------|---------|-------------|
+| `judge.tone(response)` | `str` | empathetic, professional, aggressive, neutral |
+| `judge.no_hallucination(response, context?)` | `bool` | True if no hallucination |
+| `judge.contains_action(response, action)` | `bool` | Response mentions/implies the action |
+| `judge.relevance(response, query)` | `float` | 0.0–1.0 relevance |
+| `judge.toxicity(response)` | `bool` | True if toxic |
+| `judge.faithfulness(response, source)` | `float` | 0.0–1.0 faithfulness |
+| `judge.conciseness(response)` | `str` | too_short, good, too_long |
+| `judge.score(response, criteria)` | `float` | Custom 0.0–1.0 score |
+| `judge.compare(a, b, criteria)` | `str` | "a", "b", or "tie" |
+
+---
+
+## CI in 4 Lines
+
+```yaml
+# .github/workflows/agenttest.yml
+- run: pip install agenttest-py-py
+- run: agenttest run
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+Every PR shows whether your agent got better or worse.
+
+---
+
+## agenttest vs Braintrust / DeepEval / Promptfoo
+
+| | agenttest | Braintrust | DeepEval | Promptfoo |
+|---|:---:|:---:|:---:|:---:|
+| **No account required** | ✅ | ❌ | ❌ | ❌ |
+| **No vendor lock-in** | ✅ | ❌ | ❌ | ❌ |
+| **Lives in your codebase** | ✅ | ❌ | ❌ | ❌ |
+| **Behavior diff (before/after)** | ✅ | ❌ | ❌ | ❌ |
+| **Runs locally** | ✅ | ✅ | ✅ | ✅ |
+| **MIT License** | ✅ | ❌ | ❌ | ✅ |
+| **Code-first API** | ✅ | ⚠️ | ⚠️ | ⚠️ |
+
+**agenttest** = pytest for agents. No dashboards. No SaaS. Your tests, your repo, your CI.
+
+---
+
+## Config
+
+```toml
+# agenttest.toml
+[agenttest]
+model = "claude-3-5-haiku-latest"
+timeout_seconds = 30
+workers = 4
+fail_threshold = 0.8
+cache = true
+
+[agenttest.env]
+ANTHROPIC_API_KEY = "$ANTHROPIC_API_KEY"
+```
+
+---
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+MIT

agenttest_py-0.1.0/README.md

@@ -0,0 +1,155 @@
+# agenttest
+
+**The pytest of AI agents.** Catch regressions before they reach prod.
+
+[![PyPI version](https://img.shields.io/pypi/v/agenttest-py.svg)](https://pypi.org/project/agenttest-py/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ShashStudios/agenttest/actions/workflows/agenttest.yml/badge.svg)](https://github.com/ShashStudios/agenttest/actions)
+
+---
+
+## You ship an agent. You change a prompt. Did it get better or worse?
+
+**You have no idea.** No test suite. No CI. No diff. Just deploy and hope.
+
+Every team shipping AI agents hits the same wall: your "eval" is manually pasting examples into a playground. One prompt tweak could break everything—or fix everything—and you won't know until a user complains.
+
+---
+
+## 30-Second Quickstart
+
+```bash
+pip install agenttest-py
+export ANTHROPIC_API_KEY=your_key
+agenttest init
+agenttest run
+```
+
+**Or from scratch:**
+
+```python
+# agent_test_example.py
+from agenttest import eval, judge
+
+def my_agent(query: str) -> str:
+    return "Your agent's response"  # Replace with real agent
+
+@eval
+def test_customer_support():
+    response = my_agent("I want a refund")
+    assert judge.tone(response) == "empathetic"
+    assert judge.no_hallucination(response)
+```
+
+```bash
+agenttest run
+```
+
+---
+
+## Features
+
+- **Code-first** — Tests are just Python. No YAML. No config hell.
+- **LLM-as-judge** — 9 built-in scorers: tone, hallucination, relevance, toxicity, faithfulness, conciseness, custom criteria, A/B compare.
+- **Local & CI** — Runs anywhere. Add 4 lines to GitHub Actions. No account. No dashboard.
+- **`agenttest diff`** — Side-by-side view of how your agent's responses changed between two runs. The git diff for agent behavior.
+- **Caching** — Judge results cached in `.agenttest_cache/` to avoid redundant API calls.
+- **Parallel** — `--workers 4` for faster runs.
+
+---
+
+## agenttest diff — The Git Diff for Agent Behavior
+
+See exactly how your agent's responses changed between two runs:
+
+```bash
+agenttest run --tag v1    # Before your prompt change
+agenttest run --tag v2    # After your prompt change
+agenttest diff v1 v2
+```
+
+```
+test_customer_support_refund:
+  BEFORE: "I cannot help with refunds"           pass
+  AFTER:  "I'd be happy to process that for you" pass
+  DELTA:  ✓ improved
+
+test_helpful_tone:
+  BEFORE: "Our policy states no returns"         fail
+  AFTER:  "I'm sorry to hear that. Let me help"   pass
+  DELTA:  +1 ↑
+```
+
+---
+
+## Judge Functions
+
+| Function | Returns | Description |
+|----------|---------|-------------|
+| `judge.tone(response)` | `str` | empathetic, professional, aggressive, neutral |
+| `judge.no_hallucination(response, context?)` | `bool` | True if no hallucination |
+| `judge.contains_action(response, action)` | `bool` | Response mentions/implies the action |
+| `judge.relevance(response, query)` | `float` | 0.0–1.0 relevance |
+| `judge.toxicity(response)` | `bool` | True if toxic |
+| `judge.faithfulness(response, source)` | `float` | 0.0–1.0 faithfulness |
+| `judge.conciseness(response)` | `str` | too_short, good, too_long |
+| `judge.score(response, criteria)` | `float` | Custom 0.0–1.0 score |
+| `judge.compare(a, b, criteria)` | `str` | "a", "b", or "tie" |
+
+---
+
+## CI in 4 Lines
+
+```yaml
+# .github/workflows/agenttest.yml
+- run: pip install agenttest-py-py
+- run: agenttest run
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+Every PR shows whether your agent got better or worse.
+
+---
+
+## agenttest vs Braintrust / DeepEval / Promptfoo
+
+| | agenttest | Braintrust | DeepEval | Promptfoo |
+|---|:---:|:---:|:---:|:---:|
+| **No account required** | ✅ | ❌ | ❌ | ❌ |
+| **No vendor lock-in** | ✅ | ❌ | ❌ | ❌ |
+| **Lives in your codebase** | ✅ | ❌ | ❌ | ❌ |
+| **Behavior diff (before/after)** | ✅ | ❌ | ❌ | ❌ |
+| **Runs locally** | ✅ | ✅ | ✅ | ✅ |
+| **MIT License** | ✅ | ❌ | ❌ | ✅ |
+| **Code-first API** | ✅ | ⚠️ | ⚠️ | ⚠️ |
+
+**agenttest** = pytest for agents. No dashboards. No SaaS. Your tests, your repo, your CI.
+
+---
+
+## Config
+
+```toml
+# agenttest.toml
+[agenttest]
+model = "claude-3-5-haiku-latest"
+timeout_seconds = 30
+workers = 4
+fail_threshold = 0.8
+cache = true
+
+[agenttest.env]
+ANTHROPIC_API_KEY = "$ANTHROPIC_API_KEY"
+```
+
+---
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+MIT

agenttest_py-0.1.0/agenttest/__init__.py

@@ -0,0 +1,67 @@
+"""agenttest — The pytest of AI agents."""
+
+from __future__ import annotations
+
+from .record import record
+from .assertions import (
+    assert_compare,
+    assert_conciseness,
+    assert_contains_action,
+    assert_faithfulness,
+    assert_no_hallucination,
+    assert_no_toxicity,
+    assert_relevance,
+    assert_score,
+    assert_tone,
+)
+from .judge import Judge, judge
+from .reporter import load_last_report, report_results
+from .runner import discover_tests, run_tests
+
+__all__ = [
+    "eval",
+    "judge",
+    "record",
+    "EvalResult",
+    "Judge",
+    "assert_tone",
+    "assert_contains_action",
+    "assert_no_hallucination",
+    "assert_relevance",
+    "assert_no_toxicity",
+    "assert_faithfulness",
+    "assert_conciseness",
+    "assert_score",
+    "assert_compare",
+    "discover_tests",
+    "run_tests",
+    "load_last_report",
+    "report_results",
+]
+
+
+def eval(fn: object) -> object:
+    """
+    Decorator to mark a function as an agent eval test.
+    Discovered by agenttest run.
+    """
+    # Passthrough - runner discovers by parsing source for @eval
+    return fn
+
+
+class EvalResult:
+    """Result of a single eval run."""
+
+    def __init__(
+        self,
+        test_name: str,
+        status: str,
+        duration: float = 0,
+        error_message: str | None = None,
+        scores: dict[str, float] | None = None,
+    ):
+        self.test_name = test_name
+        self.status = status
+        self.duration = duration
+        self.error_message = error_message
+        self.scores = scores or {}

agenttest_py-0.1.0/agenttest/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running as python -m agenttest."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()