agenteval-ai 0.1.0__tar.gz

agenteval_ai-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,32 @@
+name: Bug Report
+description: Report a bug in agenteval
+labels: ["bug"]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: Minimal code or steps to reproduce the issue
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+  - type: input
+    id: version
+    attributes:
+      label: agenteval version
+      placeholder: "0.1.0"
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      placeholder: "3.12"

agenteval_ai-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,17 @@
+name: Feature Request
+description: Suggest a new feature
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What problem does this solve?
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+    validations:
+      required: true

agenteval_ai-0.1.0/.github/ISSUE_TEMPLATE/new_evaluator.yml

@@ -0,0 +1,22 @@
+name: New Evaluator Proposal
+description: Propose a new built-in or community evaluator
+labels: ["evaluator", "enhancement"]
+body:
+  - type: input
+    id: name
+    attributes:
+      label: Evaluator name
+      placeholder: "toxicity"
+    validations:
+      required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: What does it catch?
+    validations:
+      required: true
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation approach
+      description: How would this evaluator work?

agenteval_ai-0.1.0/.github/pull_request_template.md

@@ -0,0 +1,14 @@
+## What
+
+<!-- Brief description of changes -->
+
+## Why
+
+<!-- What problem does this solve? -->
+
+## Checklist
+
+- [ ] Tests added/updated
+- [ ] `ruff check` passes
+- [ ] `mypy` passes
+- [ ] Documentation updated (if applicable)

agenteval_ai-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Create venv and install
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          uv pip install -e ".[all,dev]"
+
+      - name: Lint
+        run: .venv/bin/ruff check src/ tests/
+
+      - name: Type check
+        run: .venv/bin/mypy src/agenteval/
+
+      - name: Test
+        run: .venv/bin/pytest tests/unit/ -v --cov=agenteval --cov-report=xml --cov-report=term-missing
+
+      - name: Upload coverage
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          fail_ci_if_error: false

agenteval_ai-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,28 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: release
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

agenteval_ai-0.1.0/.gitignore

@@ -0,0 +1,22 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# IDE
+.idea/
+.vscode/
+.cursor/
+
+# Generated reports
+docs/report_*.html
+docs/report_*.json
+
+# Lock files
+uv.lock

agenteval_ai-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

agenteval_ai-0.1.0/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-04-09
+
+### Added
+
+- Core data models: Trace, LLMCall, ToolCall, Turn, EvalResult, TestResult, SuiteResult
+- InterceptorRegistry with auto-detection of installed SDKs
+- 5 provider interceptors: OpenAI, AWS Bedrock, Google Vertex AI, Azure OpenAI, Anthropic
+- PricingEngine with bundled pricing data for all providers
+- 13 built-in evaluators:
+  - Structural: ToolCall, Cost, Latency, LoopDetector, OutputStructure
+  - Semantic: LLMJudge, Hallucination, Similarity
+  - Safety: Security, Guardrail
+  - Operational: Regression, Convergence, ContextUtilization
+- Evaluator plugin interface with Python entry point discovery
+- Eval model providers: OpenAI and Ollama ($0 local evals)
+- pytest plugin with fixtures, markers, and CLI flags
+- Trace convenience assertions (tool_called, no_loops, no_pii_leaked, etc.)
+- CLI: `agenteval run`, `agenteval init`, `agenteval version`
+- MCP server with 8 tools + `agenteval mcp serve/install`
+- 6 cross-platform skills with adapters for Claude Code, Copilot, Cursor, Windsurf
+- 3 report formats: console (rich), HTML, JSON
+- GitHub Action with PR comment bot
+- CI/CD: GitHub Actions for testing + PyPI publishing via Trusted Publisher
+- 5 example projects: quickstart, OpenAI, Bedrock, LangChain, Ollama
+- mkdocs documentation site
+- CONTRIBUTING.md with evaluator/interceptor contribution guides

agenteval_ai-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,211 @@
+# Contributing to agenteval
+
+Thanks for your interest in contributing! agenteval is an open-source project and we welcome contributions of all kinds.
+
+## Getting Started
+
+### Development setup
+
+```bash
+# Clone the repo
+git clone https://github.com/devbrat-anand/agenteval.git
+cd agenteval
+
+# Install uv (if not already installed)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Install dev dependencies
+uv pip install -e ".[dev]" --system
+
+# Verify setup
+pytest tests/ -v
+ruff check src/ tests/
+mypy src/agenteval/
+```
+
+### Running tests
+
+```bash
+# All tests
+pytest tests/ -v
+
+# Unit tests only
+pytest tests/unit/ -v
+
+# Specific test file
+pytest tests/unit/test_models.py -v
+
+# With coverage
+pytest tests/ --cov=agenteval --cov-report=term -v
+```
+
+### Code quality
+
+```bash
+# Lint
+ruff check src/ tests/
+
+# Format
+ruff format src/ tests/
+
+# Type check
+mypy src/agenteval/
+```
+
+## Contributing an Evaluator
+
+The most impactful way to contribute. Here's how:
+
+### 1. Create the evaluator
+
+Create `src/agenteval/evaluators/your_evaluator.py`:
+
+```python
+"""Your evaluator description."""
+
+from __future__ import annotations
+
+from agenteval.core.models import EvalResult, Trace
+from agenteval.evaluators.base import Evaluator
+
+
+class YourEvaluator(Evaluator):
+    """One-line description of what this evaluator checks."""
+
+    name = "your_evaluator"
+
+    def evaluate(self, trace: Trace, criteria: dict) -> EvalResult:
+        # Your evaluation logic
+        score = 1.0
+        passed = True
+        reason = "Check passed"
+
+        # Return an EvalResult
+        return EvalResult(
+            evaluator=self.name,
+            score=score,
+            passed=passed,
+            reason=reason,
+            details={},
+        )
+```
+
+### 2. Register the entry point
+
+Add to `pyproject.toml`:
+
+```toml
+[project.entry-points."agenteval.evaluators"]
+your_evaluator = "agenteval.evaluators.your_evaluator:YourEvaluator"
+```
+
+### 3. Write tests
+
+Create `tests/unit/test_your_evaluator.py`:
+
+```python
+from agenteval.core.models import Trace, Turn
+from agenteval.evaluators.your_evaluator import YourEvaluator
+
+
+def _make_trace(**overrides) -> Trace:
+    defaults = {
+        "agent_name": "test", "input": "query", "output": "answer",
+        "turns": [], "total_cost_usd": 0.01, "total_latency_ms": 500,
+        "total_input_tokens": 100, "total_output_tokens": 50, "metadata": {},
+    }
+    defaults.update(overrides)
+    return Trace(**defaults)
+
+
+def test_your_evaluator_passes():
+    trace = _make_trace(output="good response")
+    evaluator = YourEvaluator()
+    result = evaluator.evaluate(trace)
+    assert result.passed
+    assert result.score > 0.0
+
+
+def test_your_evaluator_fails():
+    trace = _make_trace(output="bad response")
+    evaluator = YourEvaluator()
+    result = evaluator.evaluate(trace)
+    assert not result.passed
+```
+
+### 4. Submit a PR
+
+- Run `pytest tests/ -v` — all tests pass
+- Run `ruff check src/ tests/` — no lint errors
+- Run `mypy src/agenteval/` — no type errors
+- Create a PR with a clear description
+
+## Contributing an Interceptor
+
+Provider interceptors capture LLM calls at the SDK or transport level.
+
+### 1. Create the interceptor
+
+Create `src/agenteval/interceptors/your_provider.py`:
+
+```python
+"""Your provider interceptor."""
+
+from __future__ import annotations
+
+from agenteval.core.models import LLMCall
+from agenteval.interceptors.base import Interceptor
+
+
+class YourProviderInterceptor(Interceptor):
+    """Intercept calls to YourProvider SDK."""
+
+    name = "your_provider"
+
+    def install(self) -> None:
+        """Monkey-patch or hook into the provider SDK."""
+        # Hook into the SDK here
+        pass
+
+    def uninstall(self) -> None:
+        """Restore original SDK behavior."""
+        pass
+```
+
+### 2. Register in InterceptorRegistry
+
+Add the auto-detect logic in `src/agenteval/interceptors/base.py`.
+
+### 3. Add pricing data
+
+Add model pricing to `src/agenteval/interceptors/pricing.json`.
+
+## Issue Labels
+
+| Label | Use for |
+|---|---|
+| `bug` | Something is broken |
+| `enhancement` | New feature or improvement |
+| `evaluator` | New evaluator proposal |
+| `good-first-issue` | Good for newcomers |
+| `documentation` | Documentation improvements |
+
+## Code Style
+
+- Python 3.10+, type hints everywhere
+- `ruff` for linting and formatting
+- `mypy` strict mode on public API
+- Docstrings on all public functions
+- Tests for all new code (TDD preferred)
+
+## Pull Request Checklist
+
+- [ ] Tests added/updated
+- [ ] `ruff check` passes
+- [ ] `mypy` passes
+- [ ] Documentation updated (if applicable)
+- [ ] Commit messages follow conventional commits
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the MIT License.

agenteval_ai-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Devbrat Anand
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agenteval_ai-0.1.0/Makefile

@@ -0,0 +1,44 @@
+.PHONY: install dev test lint typecheck format clean build publish docs
+
+install:  ## Install agenteval with all extras
+	uv pip install -e ".[all]"
+
+dev:  ## Install with dev dependencies
+	uv pip install -e ".[all,dev]"
+
+test:  ## Run tests with coverage
+	pytest tests/ -v --cov=agenteval --cov-report=term-missing
+
+test-unit:  ## Run unit tests only
+	pytest tests/unit/ -v
+
+lint:  ## Run linter
+	ruff check src/ tests/
+
+typecheck:  ## Run type checker
+	mypy src/agenteval/
+
+format:  ## Format code
+	ruff format src/ tests/
+	ruff check --fix src/ tests/
+
+clean:  ## Remove build artifacts
+	rm -rf build/ dist/ *.egg-info .pytest_cache .mypy_cache .ruff_cache
+	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+
+build:  ## Build package
+	uv build
+
+publish: build  ## Publish to PyPI (use trusted publisher in CI)
+	uv publish
+
+docs:  ## Serve docs locally
+	mkdocs serve
+
+docs-build:  ## Build docs
+	mkdocs build
+
+help:  ## Show this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
+
+.DEFAULT_GOAL := help