agentassert 0.1.0__py3-none-any.whl

agentassert-0.1.0.dist-info/METADATA

@@ -0,0 +1,395 @@
+Metadata-Version: 2.4
+Name: agentassert
+Version: 0.1.0
+Summary: Behavioral testing framework for AI agents — pytest for AI agents
+Project-URL: Homepage, https://github.com/kaushikdhola/agentunit
+Project-URL: Documentation, https://github.com/kaushikdhola/agentunit#readme
+Project-URL: Repository, https://github.com/kaushikdhola/agentunit.git
+Project-URL: Issues, https://github.com/kaushikdhola/agentunit/issues
+Author: Kaushik Dhola
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,ai,autogen,behavioral-testing,crewai,langchain,llm,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: pluggy>=1.3.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.0.0
+Provides-Extra: all-adapters
+Requires-Dist: agentunit[autogen,crewai,langchain,llamaindex]; extra == 'all-adapters'
+Provides-Extra: autogen
+Requires-Dist: pyautogen>=0.4; extra == 'autogen'
+Provides-Extra: crewai
+Requires-Dist: crewai>=0.80; extra == 'crewai'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.8.0; extra == 'dev'
+Requires-Dist: pre-commit>=3.6.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.3.0; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3; extra == 'langchain'
+Requires-Dist: langchain>=0.3; extra == 'langchain'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index>=0.12; extra == 'llamaindex'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="https://img.shields.io/badge/AgentUnit-v0.1.0-blue?style=for-the-badge" alt="Version">
+  <img src="https://img.shields.io/badge/python-3.10+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python">
+  <img src="https://img.shields.io/badge/license-MIT-orange?style=for-the-badge" alt="License">
+</p>
+
+<h1 align="center">AgentUnit</h1>
+
+<p align="center">
+  <strong>The Behavioral Testing Framework for AI Agents</strong>
+</p>
+
+<p align="center">
+  <em>"pytest for AI Agents" — Write tests. Run agents. Ship with confidence.</em>
+</p>
+
+<p align="center">
+  <a href="#quick-start">Quick Start</a> •
+  <a href="#features">Features</a> •
+  <a href="#documentation">Documentation</a> •
+  <a href="#contributing">Contributing</a>
+</p>
+
+---
+
+## The Problem
+
+Teams building AI agents face a critical gap in their development workflow:
+
+| What Exists | What's Missing |
+|-------------|----------------|
+| Observability (LangSmith, Langfuse) | Behavioral test runners |
+| Evaluation dashboards | CI/CD pass/fail gates |
+| LLM output quality metrics | Agent execution path testing |
+
+**AgentUnit fills this gap.** It's an open-source, framework-agnostic, local-first behavioral test framework designed specifically for AI agent pipelines.
+
+## Why AgentUnit?
+
+- **Framework Agnostic** — Works with LangChain, CrewAI, AutoGen, LlamaIndex, or raw API calls
+- **Local First** — No cloud accounts, no dashboards, no external services required
+- **CI/CD Native** — Designed for `git push → test → deploy` workflows
+- **Deterministic** — Seeded execution for reproducible test runs
+- **Developer Friendly** — Familiar pytest-like syntax and workflow
+
+---
+
+## Quick Start
+
+### Installation
+
+```bash
+pip install agentassert
+```
+
+### Write Your First Test
+
+```python
+# tests/test_my_agent.py
+from agentunit import agent_test, expect, mock_tool, contains
+
+@agent_test
+def test_research_agent_workflow(agent_harness):
+    """Test that the research agent follows the correct tool sequence."""
+    
+    # 1. Create mock tools with deterministic responses
+    search = mock_tool("web_search", returns={"results": ["AI breakthrough news"]})
+    summarize = mock_tool("summarize", returns="Key finding: AI is advancing rapidly")
+
+    # 2. Run your agent under test
+    trace = agent_harness.run(
+        agent=my_research_agent,
+        input="Find the latest AI news",
+        tools=[search, summarize]
+    )
+
+    # 3. Assert on behavioral expectations
+    expect(trace).tool("web_search").was_called()
+    expect(trace).tool("web_search").called_before("summarize")
+    expect(trace).tool("web_search").called_with(query=contains("AI"))
+    expect(trace).completed_within_steps(10)
+    expect(trace).output.not_empty()
+```
+
+### Run Your Tests
+
+```bash
+$ agentunit run tests/
+
+AgentUnit v0.1.0 — Behavioral Testing Framework for AI Agents
+collecting ... 3 tests
+
+tests/test_my_agent.py
+  ✓ test_research_agent_workflow        (2 steps, $0.002, 0.3s)
+  ✓ test_handles_api_failure            (1 steps, $0.001, 0.1s)
+  ✓ test_stays_within_budget            (4 steps, $0.008, 0.5s)
+
+════════════════════════════════════════════════════════════
+3 passed in 0.9s
+════════════════════════════════════════════════════════════
+```
+
+---
+
+## Features
+
+### Mock Tools
+
+Create deterministic tool responses for predictable testing:
+
+```python
+# Static response
+search = mock_tool("web_search", returns={"results": ["item1", "item2"]})
+
+# Sequential responses
+api = mock_tool("api_call", returns_sequence=[
+    {"status": "pending"},
+    {"status": "processing"},
+    {"status": "complete"}
+])
+
+# Simulate failures
+flaky_api = mock_tool("external_service", raises=ConnectionError("timeout"))
+
+# Rate limiting simulation
+limited_api = mock_tool("rate_limited_api", returns="ok", fail_after=5)
+```
+
+### Behavioral Assertions
+
+Assert on how your agent behaves, not just what it outputs:
+
+```python
+# Tool invocation assertions
+expect(trace).tool("search").was_called()
+expect(trace).tool("search").was_not_called()
+expect(trace).tool("search").called_exactly(3)
+expect(trace).tool("search").called_at_least(1)
+
+# Execution order assertions
+expect(trace).tool("fetch_data").called_before("process_data")
+expect(trace).tool("cleanup").called_after("main_task")
+
+# Argument matching with flexible matchers
+expect(trace).tool("api").called_with(
+    query=contains("search term"),
+    limit=greater_than(0),
+    filters=has_key("category")
+)
+
+# Execution behavior assertions
+expect(trace).completed()
+expect(trace).completed_within_steps(15)
+expect(trace).failed_gracefully()
+
+# Output assertions
+expect(trace).output.not_empty()
+expect(trace).output.contains("success")
+expect(trace).output.is_valid_json()
+```
+
+### Rich Matchers
+
+Flexible matchers for complex assertion scenarios:
+
+```python
+from agentunit import (
+    # String matchers
+    contains, matches, starts_with, ends_with, any_string,
+    
+    # Numeric matchers
+    greater_than, less_than, between,
+    
+    # Collection matchers
+    has_key, has_length, contains_item,
+    
+    # Logical matchers
+    all_of, any_of, not_, anything
+)
+
+# Combine matchers for precise assertions
+expect(trace).tool("search").called_with(
+    query=all_of(
+        starts_with("user:"),
+        contains("search"),
+        not_(contains("admin"))
+    )
+)
+```
+
+---
+
+## CLI Reference
+
+```bash
+# Run all tests in current directory
+agentunit run
+
+# Run tests in a specific directory
+agentunit run tests/
+
+# Run a specific test file
+agentunit run tests/test_research_agent.py
+
+# Filter tests by keyword
+agentunit run -k "search"
+
+# Verbose output with full tracebacks
+agentunit run -v
+
+# Minimal output (dots only)
+agentunit run -q
+
+# Set random seed for reproducibility
+agentunit run --seed 12345
+
+# Stop on first failure
+agentunit run -x
+```
+
+---
+
+## Framework Integration
+
+AgentUnit is designed to work with any agent framework:
+
+<table>
+<tr>
+<td width="50%">
+
+**LangChain**
+```python
+@agent_test
+def test_langchain_agent(agent_harness):
+    from langchain.agents import AgentExecutor
+    
+    trace = agent_harness.run(
+        agent=my_langchain_agent,
+        input="Analyze this data"
+    )
+    expect(trace).completed()
+```
+
+</td>
+<td width="50%">
+
+**Custom Agents**
+```python
+@agent_test
+def test_custom_agent(agent_harness):
+    def my_agent(prompt, tools):
+        # Your custom logic
+        return result
+    
+    trace = agent_harness.run(
+        agent=my_agent,
+        input="Process request"
+    )
+    expect(trace).completed()
+```
+
+</td>
+</tr>
+</table>
+
+---
+
+## CI/CD Integration
+
+### GitHub Actions
+
+```yaml
+name: Agent Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      
+      - name: Install dependencies
+        run: |
+          pip install agentassert
+          pip install -r requirements.txt
+      
+      - name: Run agent tests
+        run: agentunit run tests/ -v
+```
+
+### GitLab CI
+
+```yaml
+agent-tests:
+  image: python:3.11
+  script:
+    - pip install agentassert
+    - agentunit run tests/
+```
+
+---
+
+## Comparison with Alternatives
+
+| Feature | AgentUnit | LangSmith | Langfuse | DeepEval |
+|---------|-----------|-----------|----------|----------|
+| Local execution | Yes | No | Partial | Yes |
+| No account required | Yes | No | No | Yes |
+| Framework agnostic | Yes | No | Yes | Yes |
+| Behavioral assertions | Yes | No | No | No |
+| Tool call testing | Yes | Partial | Partial | No |
+| CI/CD native | Yes | Partial | Partial | Yes |
+| Deterministic replay | Yes | No | No | No |
+
+---
+
+## Documentation
+
+- **[Sample Tests](tests/test_sample.py)** — Working examples to get started
+- **[Contributing Guide](CONTRIBUTING.md)** — How to contribute to the project
+
+---
+
+## Contributing
+
+We welcome contributions from the community. Please read our [Contributing Guidelines](CONTRIBUTING.md) before submitting PRs.
+
+---
+
+## License
+
+AgentUnit is released under the **MIT License**. See [LICENSE](LICENSE) for details.
+
+---
+
+<p align="center">
+  <strong>Built by <a href="https://github.com/kaushikdhola">Kaushik Dhola</a></strong>
+</p>
+
+<p align="center">
+  <code>pip install agentassert</code>
+</p>

agentassert-0.1.0.dist-info/RECORD

@@ -0,0 +1,35 @@
+agentunit/__init__.py,sha256=ykv-40E7xGuiGRFP6dlyuvFqguWAzLDfjSKKSVVSpE8,2154
+agentunit/_version.py,sha256=aEtcAlwTYV6zYLWc8oarN4Co2KKHe167d4i5OirqLEg,129
+agentunit/decorators.py,sha256=TWvkjDR3tddWHjY9DKZLJXqg7nzcFC5dzNh9f6zrcHc,1888
+agentunit/fixtures.py,sha256=x6Q6SfWnuI5ZxX-0xRX15y_nywLOkTCu7-Kbe3LnqyI,3061
+agentunit/adapters/__init__.py,sha256=n6EL9g7m9E9rbORV1xWrFffeuytzVK9hsLev7D47xsY,188
+agentunit/adapters/base.py,sha256=kJIP5F1UrvKCdgzAzIOmfEnl_lb1LfOh5W54j0OF-mU,963
+agentunit/adapters/generic.py,sha256=GmqyxsaVtT8k_KLD7p7SRmrpajGWSnTot-qtedS1eo0,3715
+agentunit/assertions/__init__.py,sha256=JltpJE75KlRy1Kf-M22D3tuEFeLJIPedSCBnjCdR598,1028
+agentunit/assertions/behavior.py,sha256=a2bHrLAKDbjCZ-wFMkaJQQT36cbN3FuUWxnr9PzfSHM,7708
+agentunit/assertions/fluent.py,sha256=CxjgdTTZUkdmT895vDeuEONb3sqzALULN8KRW_8XpEY,7969
+agentunit/assertions/matchers.py,sha256=oahS5WL7unZIlvF1SfXePytCykNd51nSVEQj4HIOJS0,8717
+agentunit/cli/__init__.py,sha256=8K6ZMmLxeYtHL5H9Tyl5EDp87R6Hcgi43Ikyth-5CTU,90
+agentunit/cli/main.py,sha256=69kCeRouTue9gpM1bh5AVu5QOHpzoCgqry8dA_bfHo0,721
+agentunit/cli/commands/__init__.py,sha256=C3Dh-Nk8xbxPf_yNdlmsApeOqWqHOsox3wCRm3kT7lk,91
+agentunit/cli/commands/run.py,sha256=g9qaLo5-Rnr8koq4I2oFxofusIlbCU2i5lJurvmioXA,3078
+agentunit/core/__init__.py,sha256=5rT3qGBsZH1HCOh2gHINxn-5ng_7iXN4IqFugXPcvLw,416
+agentunit/core/collector.py,sha256=Si29fUOnTwIuo1NGG2khtoT3zinLpv4-xvbVp30CmC8,8750
+agentunit/core/item.py,sha256=-mGJuQEhKLFcbS2zinfBG49HKoCfDVNdJOydV07t_7E,2453
+agentunit/core/outcome.py,sha256=m6pMZfE1bYMxlGGdFQOYqlwl6nE8aAdsxxO12UpAveI,3816
+agentunit/core/runner.py,sha256=OYwYxz7p0GX4VswqJ0nrHNO9-zbSN3b3iGKyUCvRppk,5797
+agentunit/core/session.py,sha256=fHSI356YzdYX1BjCDvs2PhhJD752yeaWnuRxh1_iFTk,4261
+agentunit/mocks/__init__.py,sha256=9_ntv4aYwYHvJnNJw4WxhuyRoBKnT63eMrzTYNgOvrk,198
+agentunit/mocks/tool_mock.py,sha256=dUkeq1qgRtJNaHHTll6hILGXnlnkJlR1fpuhmKTBo3w,10136
+agentunit/reporters/__init__.py,sha256=O4HLkliIBGD24YApL_-h6wjV7g6KSx51OTujvDy4qIo,136
+agentunit/reporters/terminal.py,sha256=3uzaUOMkMuRtZFpz08TbK4ITJYKQCx0V17uAIXzr1gI,7223
+agentunit/trace/__init__.py,sha256=vtIOGUOUqYfaW-OKhcD1JiSItEpeLOz_BPpVLnW1bDA,438
+agentunit/trace/event.py,sha256=hKk9it2F6oJ47A8NMxSgazmVfT-kUBKxypaPD1ghAoU,3259
+agentunit/trace/span.py,sha256=J5F7iqIMMLN_ZXGQp9vT0JSjidoeVA6x0KavBAgz5Rc,2362
+agentunit/trace/tracer.py,sha256=cHAm4d0lIE5PJkjEWZCLPA6tcSycfEOlSgSauAZGfrQ,9724
+agentunit/trace/tree.py,sha256=6bm0SroaYoGlb-5Y1cptZBTbscCEEl-IoYSPs7uNrhQ,5203
+agentassert-0.1.0.dist-info/METADATA,sha256=N5AuGjBO455tDzP8ZC1_wG6oNa6sOQBzp-PZ1RoSizw,10542
+agentassert-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+agentassert-0.1.0.dist-info/entry_points.txt,sha256=5GCpgYiuc0Wul7fKLxB1uwTrlXwHSkm-uMlkduGf2_Y,54
+agentassert-0.1.0.dist-info/licenses/LICENSE,sha256=5Wo7S054PlecufN8LEvp6DaoR1-gS5BrDQ97d3hIE_E,1441
+agentassert-0.1.0.dist-info/RECORD,,

agentassert-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agentassert-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+agentunit = agentunit.cli.main:main

agentassert-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,34 @@
+MIT License
+
+Copyright (c) 2026 Kaushik Dhola
+
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+ATTRIBUTION NOTICE:
+
+When using AgentUnit in your projects, please provide appropriate credit by:
+1. Retaining this license file in any distribution
+2. Acknowledging AgentUnit in your project documentation
+3. Not misrepresenting the origin of this software
+
+For commercial support or enterprise licensing inquiries, please contact the author.

agentunit/__init__.py

@@ -0,0 +1,102 @@
+"""
+AgentUnit — Behavioral testing framework for AI agents.
+
+AgentUnit is the missing test runner for AI agents — framework-agnostic,
+locally executable, and built for CI/CD pipelines.
+
+Example usage:
+    import agentunit as au
+    from agentunit import agent_test, expect, mock_tool
+
+    @agent_test
+    def test_research_agent_calls_search(agent_harness):
+        search = mock_tool("web_search", returns={"results": ["AI news"]})
+
+        trace = agent_harness.run(
+            agent=my_agent,
+            input="Find AI news",
+            tools=[search]
+        )
+
+        expect(trace).tool("web_search").was_called()
+        expect(trace).completed_within_steps(10)
+"""
+
+from agentunit._version import __version__, __version_info__
+
+# Decorators
+from agentunit.decorators import agent_test, fixture, scenario, scenarios
+
+# Fluent assertions
+from agentunit.assertions.fluent import expect
+
+# Mocks
+from agentunit.mocks.tool_mock import mock_tool, MockTool, MockToolset
+
+# Matchers (for called_with assertions)
+from agentunit.assertions.matchers import (
+    contains,
+    matches,
+    starts_with,
+    ends_with,
+    any_string,
+    greater_than,
+    less_than,
+    between,
+    has_key,
+    has_length,
+    contains_item,
+    is_type,
+    is_not_none,
+    is_none,
+    anything,
+    all_of,
+    any_of,
+    not_,
+)
+
+# Core types for type hints
+from agentunit.trace.tracer import AgentTrace
+from agentunit.core.outcome import Outcome, OutcomeStatus
+from agentunit.fixtures import AgentHarness
+
+__all__ = [
+    # Version
+    "__version__",
+    "__version_info__",
+    # Decorators
+    "agent_test",
+    "fixture",
+    "scenario",
+    "scenarios",
+    # Assertions
+    "expect",
+    # Mocks
+    "mock_tool",
+    "MockTool",
+    "MockToolset",
+    # Matchers
+    "contains",
+    "matches",
+    "starts_with",
+    "ends_with",
+    "any_string",
+    "greater_than",
+    "less_than",
+    "between",
+    "has_key",
+    "has_length",
+    "contains_item",
+    "is_type",
+    "is_not_none",
+    "is_none",
+    "anything",
+    "all_of",
+    "any_of",
+    "not_",
+    # Core types
+    "AgentTrace",
+    "AgentHarness",
+    "Outcome",
+    "OutcomeStatus",
+]

agentunit/_version.py

@@ -0,0 +1,4 @@
+"""Version information for AgentUnit."""
+
+__version__ = "0.1.0"
+__version_info__ = tuple(int(x) for x in __version__.split("."))

agentunit/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Framework adapters for AgentUnit."""
+
+from agentunit.adapters.base import BaseAdapter
+from agentunit.adapters.generic import GenericAdapter
+
+__all__ = ["BaseAdapter", "GenericAdapter"]

agentunit/adapters/base.py

@@ -0,0 +1,33 @@
+"""Base adapter interface for framework integrations."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from agentunit.trace.tracer import AgentTrace
+
+
+class BaseAdapter(ABC):
+    """
+    Base interface for framework adapters.
+
+    Adapters bridge AgentUnit's generic harness and tracer with framework-specific
+    execution models.
+    """
+
+    @abstractmethod
+    def wrap_agent(self, agent: Any) -> Any:
+        """Wrap an agent object with adapter-specific instrumentation."""
+
+    @abstractmethod
+    def run_agent(self, agent: Any, input: Any, **kwargs: Any) -> Any:
+        """Execute the agent and return its output."""
+
+    @abstractmethod
+    def extract_trace(self) -> AgentTrace:
+        """Extract and return the execution trace."""
+
+    @abstractmethod
+    def inject_mock_tools(self, agent: Any, tools: list[Any]) -> Any:
+        """Replace real tools with mock tools on the given agent object."""