agenteval-py 0.1.0__tar.gz

agenteval_py-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: uv pip install -e ".[dev,all]" --system
+
+      - name: Lint
+        run: ruff check src/ tests/ --select F,I
+
+      - name: Test
+        run: pytest tests/ -v --tb=short
+
+      - name: Build package
+        run: python -m build

agenteval_py-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,30 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build
+        run: python -m pip install --upgrade build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

agenteval_py-0.1.0/.gitignore

@@ -0,0 +1,104 @@
+# ----------------------
+# Python basics
+# ----------------------
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# ----------------------
+# Virtual environments
+# ----------------------
+.venv/
+venv/
+env/
+ENV/
+
+# uv-specific (lock + cache)
+.uv/
+uv.lock
+
+# ----------------------
+# Distribution / packaging
+# ----------------------
+build/
+dist/
+*.egg-info/
+.eggs/
+pip-wheel-metadata/
+
+# ----------------------
+# Logs & runtime files
+# ----------------------
+*.log
+logs/
+*.pid
+*.seed
+
+# ----------------------
+# Environment variables
+# ----------------------
+.env
+.env.*
+*.env
+
+# ----------------------
+# IDE / Editor
+# ----------------------
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# ----------------------
+# macOS
+# ----------------------
+.DS_Store
+
+# ----------------------
+# Testing
+# ----------------------
+.pytest_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# ----------------------
+# Type checking
+# ----------------------
+.mypy_cache/
+.pyre/
+
+# ----------------------
+# Ruff / linting
+# ----------------------
+.ruff_cache/
+
+# ----------------------
+# Jupyter
+# ----------------------
+.ipynb_checkpoints/
+
+# ----------------------
+# CLI / Agent artifacts
+# ----------------------
+outputs/
+runs/
+artifacts/
+cache/
+tmp/
+
+# ----------------------
+# Secrets / API keys
+# ----------------------
+secrets.json
+config.local.json
+
+# ----------------------
+# Misc
+# ----------------------
+*.sqlite3
+*.db

agenteval_py-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.360
+    hooks:
+      - id: pyright
+        stages: [manual]

agenteval_py-0.1.0/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+All notable changes to agenteval are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/).
+
+---
+
+## [Unreleased]
+
+Nothing yet.
+
+---
+
+## [0.1.0] — 2024-01-01
+
+Initial release.
+
+### Added
+
+- **Tracer** — records tool calls (name, arguments, result, duration, error) and run boundaries via `tracer.wrap()`, `@tracer.tool`, and `async with tracer.run()`
+- **AssertionSet** — fluent, chainable assertion API with collected failures: `called_tool`, `never_called_tool`, `tool_call_count`, `tool_called_before`, `tool_called_with_args`, `completed_within_steps`, `completed_within_seconds`, `response_contains`, `response_matches_schema`, `no_errors`, `custom`
+- **Runner** — executes a test function N times concurrently using `anyio`, supports both sync and async test functions
+- **`@agenteval.test` decorator** — registers test functions with `n`, `threshold`, and `tags` parameters; supports bare and parameterized form
+- **Suite runner** — discovers `test_*.py` files, imports them, and runs all registered tests
+- **RichReporter** — color-coded terminal output (✅ ⚠️ ❌) with pass rate, timing, step count, and optional per-run trace details; JSON export for CI
+- **CLI** — `agenteval run` and `agenteval report` commands via Typer
+- **OpenAI adapter** — `wrap_tools()` and `extract_token_usage()` for OpenAI function calling
+- **Anthropic adapter** — `wrap_tools()` and `extract_token_usage()` for Anthropic tool use (including cache token fields)
+- **LangChain adapter** — `AgentEvalCallbackHandler` that auto-connects to the active `Tracer` via `ContextVar`, enabling concurrent runs with full isolation
+- **Data models** — `AgentTrace`, `ToolCall`, `TestResult`, `SuiteResult` — all Pydantic v2, fully serializable to JSON
+- **Typed** — `py.typed` marker included for downstream type checkers
+- **CI** — GitHub Actions workflow testing Python 3.11, 3.12, and 3.13

agenteval_py-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,130 @@
+# Contributing to agenteval
+
+Thanks for taking the time to contribute. This document covers everything you need to get the development environment running, the conventions the codebase follows, and what a good pull request looks like.
+
+---
+
+## Getting set up
+
+You'll need Python 3.11 or later. The project uses `pip` for dependency management and `hatchling` as the build backend.
+
+```bash
+git clone https://github.com/awesome-pro/agenteval
+cd agenteval
+
+# Install in editable mode with all dev dependencies
+pip install -e ".[dev]"
+
+# Install pre-commit hooks (runs ruff on every commit automatically)
+pre-commit install
+```
+
+That's it. Run the tests to make sure everything is working:
+
+```bash
+pytest tests/ -v
+```
+
+All tests pass without any API keys. The framework is tested against mock agents.
+
+---
+
+## Project layout
+
+```
+src/agenteval/       — library source code
+tests/               — test suite (mirrors src/agenteval/ structure)
+docs/                — topic-based documentation
+.github/workflows/   — CI configuration (lint + tests + build on 3.11/3.12/3.13)
+```
+
+The source lives under `src/` to keep it cleanly separated from tests and config. When you add a new module, mirror it in `tests/` with a `test_` prefix.
+
+---
+
+## Code conventions
+
+**Type annotations** — the package ships with type information and uses Pyright
+for type checking. If you're working on typing-heavy changes, run `pyright src/`
+before pushing.
+
+**Formatting and linting** — Ruff handles both. The pre-commit hook runs it automatically on changed files. You can also run it manually:
+
+```bash
+ruff check src/ tests/ --select F,I
+ruff format src/
+```
+
+**Async first** — the runner, tracer, and suite are all async-native. If you're adding functionality that touches the execution path, prefer `async def` and `anyio` primitives over `asyncio` directly. This keeps things backend-agnostic.
+
+**No framework dependencies in core** — `src/agenteval/` (outside `adapters/`) should not import `openai`, `anthropic`, or `langchain`. Framework-specific code belongs in `src/agenteval/adapters/`.
+
+**Keep the public API surface small** — `__init__.py` exports only what users actually need. If something is implementation detail, don't add it to `__all__`.
+
+---
+
+## Running specific test groups
+
+```bash
+# Run everything
+pytest tests/ -v
+
+# Run a specific file
+pytest tests/test_tracer.py -v
+
+# Run with a keyword filter
+pytest tests/ -k "assertion" -v
+
+# Run adapter tests only
+pytest tests/adapters/ -v
+```
+
+---
+
+## Making changes
+
+**For bug fixes:** open an issue first if the behavior is surprising or the fix is non-obvious. Small, clear bugs can go straight to a PR.
+
+**For new features:** please open an issue or discussion first before building anything significant. This saves everyone time if the design needs to change.
+
+**For documentation:** PRs that improve clarity, fix typos, or add examples are always welcome without prior discussion.
+
+---
+
+## Pull request checklist
+
+Before submitting, make sure:
+
+- [ ] `pytest tests/ -v` passes
+- [ ] `ruff check src/ tests/ --select F,I` passes with no errors
+- [ ] `python -m build` completes successfully
+- [ ] New behavior has corresponding tests
+- [ ] If you added a public API, it's documented in `docs/` and/or the relevant docstring
+- [ ] Commit messages are descriptive (what changed and roughly why, not just "fix bug")
+
+---
+
+## Commit style
+
+No strict convention here, but a good commit message answers two questions: what changed, and why. One-liners are fine for small changes:
+
+```
+fix: tool wrapper now records arguments for positional-only params
+```
+
+For anything that took real thought, add a short body:
+
+```
+feat: add response_matches_schema assertion
+
+Agents returning structured JSON are common enough that a schema
+validation assertion makes sense as a first-class feature. Uses
+Pydantic v2 model_validate under the hood, which also handles
+JSON string inputs automatically.
+```
+
+---
+
+## Questions
+
+If something in the codebase is confusing or the docs are unclear, opening an issue to ask is completely fine. Clear documentation is a feature.

agenteval_py-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Abhinandan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.