evalgrid 0.1.0__tar.gz

evalgrid-0.1.0/.env.example

@@ -0,0 +1,2 @@
+ANTHROPIC_API_KEY=sk-ant-your-key-here
+OPENAI_API_KEY=sk-your-key-here-optional

evalgrid-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+      - name: Verify CLI
+        run: evalgrid --help

evalgrid-0.1.0/.github/workflows/evalgrid.yml

@@ -0,0 +1,59 @@
+name: evalgrid
+
+on:
+  workflow_dispatch:
+
+jobs:
+  eval:
+    name: Run Evals
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Install evalgrid
+        run: pip install -e ".[dev]"
+
+      - name: Initialize evalgrid
+        run: evalgrid init
+
+      - name: Copy example scenarios
+        run: |
+          if [ -d examples ]; then
+            cp examples/*.yml .evalgrid/scenarios/ 2>/dev/null || true
+          fi
+
+      - name: Run Evals (mock mode — no API key required)
+        run: |
+          evalgrid run \
+            --mock \
+            --output results.json \
+            --config .evalgrid/config.yml
+
+      - name: Run Evals with real API (optional)
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          if [ -n "$ANTHROPIC_API_KEY" ]; then
+            evalgrid run \
+              --output results-real.json \
+              --config .evalgrid/config.yml
+          else
+            echo "Skipping real API run — ANTHROPIC_API_KEY not configured"
+          fi
+
+      - name: Upload Results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: evalgrid-results-${{ github.run_id }}
+          path: results.json
+          retention-days: 30
+

evalgrid-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,72 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+  build:
+    name: Build distribution
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/evalgrid/
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

evalgrid-0.1.0/.gitignore

@@ -0,0 +1,72 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+# Databases (dev artifacts)
+*.db
+*.sqlite
+
+# evalgrid runtime
+.evalgrid/
+.agenteval/
+sdr-demo-result.json
+sdr-real-result.json
+*-results.json
+*-result.json
+CLAUDE.md
+.claude/
+.private/
+
+# Node / dashboard
+dashboard/node_modules/
+dashboard/dist/
+dashboard/test-results/
+dashboard/Scenario
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Environment / secrets
+.env
+.env.local
+.env.*.local
+
+# Node (general)
+node_modules/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Build artifacts
+*.whl
+*.tar.gz
+
+# mypy cache
+.mypy_cache/
+
+# ruff cache
+.ruff_cache/
+templates/

evalgrid-0.1.0/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+All notable changes to evalgrid will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [0.1.0] - 2026-05-06
+
+Initial release.
+
+### Added
+
+- `evalgrid init` — scaffold `.evalgrid/` config and example scenario
+- `evalgrid run` — run all scenarios and print pass/fail results
+- `evalgrid run --mock` — dry run with no API calls (CI-friendly)
+- `evalgrid run --tag <tag>` — filter scenarios by tag
+- `evalgrid run --output <file>` — save results to JSON
+- `evalgrid scenario add` — interactively create a new scenario
+- `evalgrid scenario list` — list all scenarios in the scenarios directory
+- `evalgrid scenario validate` — validate scenario YAML files
+- `evalgrid server` — start the React dashboard at `http://localhost:8000`
+- FastAPI REST API: `GET /api/runs`, `POST /api/runs`, `GET /api/runs/{id}`, `GET /api/scenarios`, `POST /api/scenarios`
+- SQLite persistence via SQLAlchemy async (results survive server restarts)
+- React dashboard: pass rate ring chart, avg latency sparkline, run history, per-step score breakdowns
+- Dark/light mode, mobile responsive dashboard
+- LLM-as-judge scoring via litellm (supports Anthropic, OpenAI, Google, Ollama)
+- Mock scorer for CI runs that require no API key
+- 8 example scenarios: Claude Code suite (5 scenarios), hello-world, sdr-qualify-lead, support-triage
+- Python API: `EvalConfig`, `ScenarioLoader`, `ScenarioRunner`

evalgrid-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,82 @@
+# Contributing to evalgrid
+
+Thank you for your interest in contributing to evalgrid.
+
+## Prerequisites
+
+- Python 3.11 or newer
+- Node.js 18+ (only if working on the dashboard)
+- Git
+
+## Dev Setup
+
+```bash
+git clone https://github.com/naman006-rai/evalgrid.git
+cd evalgrid
+pip install -e ".[dev]"
+```
+
+This installs evalgrid in editable mode along with test dependencies (pytest, ruff, mypy).
+
+## Running Tests
+
+```bash
+pytest tests/ -v
+```
+
+All tests must pass before submitting a pull request. The CI workflow runs the full suite automatically.
+
+To run a specific test file:
+
+```bash
+pytest tests/test_api.py -v
+```
+
+## Dashboard (optional)
+
+If you are modifying the React dashboard:
+
+```bash
+cd dashboard
+npm install
+npm run dev      # dev server at localhost:5173
+npm run build    # production build
+```
+
+After building, copy the output to the static directory:
+
+```bash
+# Windows (PowerShell)
+Copy-Item -Recurse -Force dashboard\dist\* evalgrid\static\
+
+# macOS / Linux
+cp -r dashboard/dist/* evalgrid/static/
+```
+
+## Code Style
+
+Python code is linted with `ruff` and type-checked with `mypy`:
+
+```bash
+ruff check evalgrid/
+mypy evalgrid/
+```
+
+## PR Guidelines
+
+1. Fork the repo and create a branch: `git checkout -b feat/my-feature`
+2. Write tests for your change (keep coverage above 80%)
+3. Run the full suite: `pytest tests/ -v`
+4. Run linting: `ruff check evalgrid/`
+5. Open a pull request against `main` with a clear description of what changes and why
+6. CI will run evals and tests automatically — fix any failures before requesting review
+
+## Reporting Issues
+
+Open a GitHub issue at <https://github.com/naman006-rai/evalgrid/issues>.
+
+Include:
+- evalgrid version (`pip show evalgrid`)
+- Python version (`python --version`)
+- Steps to reproduce
+- Expected vs actual behavior

evalgrid-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Naman Rai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalgrid-0.1.0/PKG-INFO

@@ -0,0 +1,263 @@
+Metadata-Version: 2.4
+Name: evalgrid
+Version: 0.1.0
+Summary: Agent evaluation framework — run, score, and track AI agent correctness at scale.
+Project-URL: Homepage, https://github.com/naman006-rai/evalgrid
+Project-URL: Documentation, https://github.com/naman006-rai/evalgrid#readme
+Project-URL: Repository, https://github.com/naman006-rai/evalgrid
+Project-URL: Issues, https://github.com/naman006-rai/evalgrid/issues
+Project-URL: Bug Tracker, https://github.com/naman006-rai/evalgrid/issues
+Author-email: Naman Rai <naman.rai006@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,evaluation,llm,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: click>=8.1.0
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: litellm<2.0.0,>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: uvicorn[standard]>=0.32.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.13.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.7.0; extra == 'dev'
+Provides-Extra: postgres
+Requires-Dist: asyncpg>=0.30.0; extra == 'postgres'
+Requires-Dist: psycopg2-binary>=2.9.0; extra == 'postgres'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<svg width="52" height="52" viewBox="0 0 36 36" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <rect x="2" y="2" width="14" height="14" rx="3" fill="#5b5ef4"/>
+  <rect x="20" y="2" width="14" height="14" rx="3" fill="#5b5ef4" opacity="0.4"/>
+  <rect x="2" y="20" width="14" height="14" rx="3" fill="#5b5ef4" opacity="0.4"/>
+  <rect x="20" y="20" width="14" height="14" rx="3" fill="#5b5ef4" opacity="0.15"/>
+  <path d="M8 9 L11 12 L16 6" stroke="white" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>
+
+# evalgrid
+
+**Agent evaluation framework — run, score, and track AI agent correctness at scale.**
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+AI agents fail in ways unit tests can't catch: the customer support agent stops calling the refund tool after a prompt change. The research agent loops on the same Google search 23 times. The reconciliation agent silently picks the wrong account format on edge cases. evalgrid catches these regressions before they ship.
+
+</div>
+
+## Status
+
+EvalGrid is in early development (v0.1). The framework runs reliably for local-first agent evaluation, but you should expect:
+
+- Some rough edges in the dashboard UI
+- Schema changes in scenario YAML between v0.1 and v0.2
+- GitHub status check / CI gate coming in v0.2
+
+If you're building agents and want eval-driven workflows now, this works. If you need production-grade CI integration, watch for v0.2.
+
+## Screenshots
+
+![Dashboard overview](docs/screenshots/dashboard-overview.png)
+*Dashboard showing run history with pass rates and per-scenario breakdowns*
+
+![Run detail](docs/screenshots/run-detail.png)
+*Per-step verdicts with judge reasoning for a failed scenario*
+
+---
+
+## Install
+
+```bash
+pip install evalgrid
+```
+
+## Quick start
+
+**1. Initialize**
+```bash
+evalgrid init
+```
+Scaffolds `.evalgrid/` with a config file and an example scenario.
+
+**2. Write a scenario**
+
+`.evalgrid/scenarios/qualify_lead.yml`:
+```yaml
+id: sdr-qualify-lead
+name: SDR Lead Qualification
+description: Verify BANT qualification on an inbound enterprise lead
+agent_type: sdr
+
+prompt: |
+  You are an expert SDR. Qualify leads using the BANT framework.
+  Identify budget, authority, need, and timeline. Recommend a next step.
+
+test_input: |
+  Prospect: Sarah Chen, VP Engineering at TechCorp (500 employees)
+  Budget: $50k allocated this quarter for developer tooling
+  She filled out a demo request form 20 minutes ago.
+
+expected_actions:
+  - action: identify_bant_criteria
+    required: true
+    description: Identify which BANT criteria are present
+  - action: ask_timeline_question
+    required: true
+    description: Ask about the buying timeline
+  - action: recommend_next_step
+    required: false
+    description: Propose a clear next step (demo, call, etc.)
+
+tags: [sdr, bant]
+```
+
+**3. Run**
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+evalgrid run
+```
+
+**4. See results**
+```
+Running 1 scenario(s)...
+
+ ✓  SDR Lead Qualification   sdr   PASS   0.94   3/3
+
+1/1 scenarios passed (100%)
+Results saved to .evalgrid/results/
+```
+
+**5. Open the dashboard**
+```bash
+evalgrid server
+```
+Then visit `http://localhost:8000` for the full run history, pass rate trends, and per-step breakdowns.
+
+> **Note:** The dashboard binds to `127.0.0.1` by default and is designed for local use only. Do not expose it to a public network.
+
+---
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `evalgrid init` | Scaffold `.evalgrid/` in your project |
+| `evalgrid run` | Run all scenarios and print pass/fail |
+| `evalgrid run --tag sdr` | Filter by tag |
+| `evalgrid run --mock` | Dry run with no API calls (great for CI) |
+| `evalgrid run --output results.json` | Save results to a file |
+| `evalgrid scenario add` | Interactively create a new scenario |
+| `evalgrid scenario list` | List all scenarios |
+| `evalgrid server` | Start the React dashboard |
+
+## Scenario format
+
+```yaml
+id: unique-scenario-id
+name: Human Readable Name
+description: What this scenario tests
+agent_type: sdr | coding | support | research | finance | legal | generic
+
+prompt: |
+  System prompt for your agent...
+
+test_input: |
+  The user message / input to evaluate...
+
+expected_actions:
+  - action: action_name
+    required: true
+    description: What this action entails
+
+success_criteria: >
+  Plain-language description of what a passing response looks like.
+
+tags: [tag1, tag2]
+timeout_seconds: 60
+```
+
+## Architecture: agents and judges
+
+EvalGrid is a runner-and-scorer framework. The agent (the LLM being tested) and the judge (the LLM doing the scoring) are configured independently and can use different providers via LiteLLM.
+
+The example scenarios that ship with v0.1 use Claude for both — that's a demo choice, not an architectural constraint. Common real-world setups include:
+
+- GPT-4o agent, Claude Sonnet judge (cross-model validation)
+- Claude Haiku agent, Claude Sonnet judge (cost-efficient testing)
+- Llama 3.1 agent, GPT-4o judge (open-source agent, frontier judge)
+
+Configure in `.evalgrid/config.yml`:
+
+```yaml
+model: gpt-4o          # the agent being tested
+judge:
+  provider: anthropic
+  model: claude-sonnet-4-6   # the model scoring the agent
+```
+
+## Examples
+
+Example scenarios are in the [`examples/`](./examples/) directory. Copy them to get started:
+
+```bash
+cp examples/*.yml .evalgrid/scenarios/
+```
+
+## GitHub Actions
+
+Add to `.github/workflows/ci.yml`:
+
+```yaml
+- name: Run evalgrid
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  run: evalgrid run --output results.json
+```
+
+## Python API
+
+```python
+from evalgrid.core import EvalConfig, ScenarioLoader, ScenarioRunner
+
+config = EvalConfig(model="claude-sonnet-4-6")
+loader = ScenarioLoader()
+scenarios = loader.load_dir(".evalgrid/scenarios")
+
+runner = ScenarioRunner(config)
+results = runner.run_all(scenarios, on_result=lambda r: print(r.scenario_name, r.status))
+
+for result in results:
+    print(f"{result.scenario_name}: {result.pass_rate:.0%}")
+```
+
+## Security
+
+evalgrid is a **local-only developer tool**. Keep these constraints in mind:
+
+- The API server binds to `127.0.0.1` by default. Use `--bind` to change this, but be aware that exposing the server to a network grants unauthenticated access to your scenarios and eval runs.
+- Scenario files are sandboxed to `~/.evalgrid/scenarios/`. Paths supplied via the API are validated and rejected if they escape this directory.
+- Provider API keys (e.g. `ANTHROPIC_API_KEY`) are read from environment variables only — never pass them through the API.
+- Do not run `evalgrid server` on a shared or public-facing machine.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for full setup instructions and PR guidelines.
+
+## License
+
+MIT. See [LICENSE](LICENSE). Copyright (c) 2026 Naman Rai.