evaldeck 0.1.0__tar.gz

evaldeck-0.1.0/.claude/settings.local.json

@@ -0,0 +1,31 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python -m pytest:*)",
+      "Bash(grep:*)",
+      "Bash(python:*)",
+      "Bash(python3:*)",
+      "Bash(PYTHONPATH=src python3:*)",
+      "Bash(PYTHONPATH=src /opt/homebrew/bin/python3.11:*)",
+      "Bash(wc:*)",
+      "WebFetch(domain:docs.langchain.com)",
+      "WebSearch",
+      "WebFetch(domain:arize-ai.github.io)",
+      "WebFetch(domain:docs.arize.com)",
+      "WebFetch(domain:arize.com)",
+      "WebFetch(domain:github.com)",
+      "Bash(ls:*)",
+      "Bash(find:*)",
+      "Bash(pip install:*)",
+      "Bash(pip3 install:*)",
+      "Bash(mkdocs serve --help:*)",
+      "Bash(git init:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(git branch:*)",
+      "Bash(git remote add:*)",
+      "Bash(git push:*)",
+      "Bash(ruff check:*)"
+    ]
+  }
+}

evaldeck-0.1.0/.devcontainer/Dockerfile

@@ -0,0 +1,30 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.11-bullseye
+
+# Remove yarn repo (has expired GPG key) and install system dependencies
+RUN rm -f /etc/apt/sources.list.d/yarn.list && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /workspaces/evaldeck
+
+# Upgrade pip
+RUN pip install --upgrade pip
+
+# Install Python development tools
+RUN pip install --no-cache-dir \
+    ruff \
+    mypy \
+    pytest \
+    pytest-asyncio \
+    pre-commit \
+    build \
+    twine
+
+# Create directory for evaldeck output
+RUN mkdir -p /workspaces/evaldeck/.evaldeck
+
+# Set Python path
+ENV PYTHONPATH="/workspaces/evaldeck/src:${PYTHONPATH}"

evaldeck-0.1.0/.devcontainer/devcontainer.json

@@ -0,0 +1,60 @@
+{
+  "name": "Evaldeck Development",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/git:1": {},
+    "ghcr.io/devcontainers/features/github-cli:1": {}
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "charliermarsh.ruff",
+        "tamasfe.even-better-toml",
+        "redhat.vscode-yaml",
+        "github.copilot"
+      ],
+      "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": [
+          "tests"
+        ],
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+          "source.organizeImports": "explicit",
+          "source.fixAll": "explicit"
+        },
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff"
+        },
+        "files.exclude": {
+          "**/__pycache__": true,
+          "**/*.pyc": true,
+          "**/.pytest_cache": true,
+          "**/.mypy_cache": true,
+          "**/.ruff_cache": true,
+          "**/*.egg-info": true
+        }
+      }
+    }
+  },
+  "postCreateCommand": "pip install -e '.[dev,all,docs]' && (git rev-parse --git-dir > /dev/null 2>&1 && pre-commit install || echo 'Skipping pre-commit install (not a git repo)')",
+  "runArgs": [
+    "--env-file",
+    "${localWorkspaceFolder}/.env"
+  ],
+  "forwardPorts": [],
+  "remoteUser": "vscode",
+  "mounts": [
+    "source=${localWorkspaceFolder}/.evaldeck,target=/workspaces/evaldeck/.evaldeck,type=bind,consistency=cached"
+  ],
+  "remoteEnv": {
+    "PYTHONDONTWRITEBYTECODE": "1",
+    "PYTHONUNBUFFERED": "1"
+  }
+}

evaldeck-0.1.0/.github/workflows/ci.yaml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint with ruff
+        run: |
+          ruff check src/ tests/
+
+      - name: Type check with mypy
+        run: |
+          mypy src/
+
+      - name: Run tests
+        run: |
+          pytest tests/ -v --tb=short
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: |
+          python -m build
+
+      - name: Check package
+        run: |
+          pip install twine
+          twine check dist/*

evaldeck-0.1.0/.github/workflows/docs.yaml

@@ -0,0 +1,50 @@
+name: Deploy Docs
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[docs]"
+
+      - name: Build docs
+        run: mkdocs build
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site/
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

evaldeck-0.1.0/.github/workflows/publish.yaml

@@ -0,0 +1,28 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # Required for trusted publishing
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

evaldeck-0.1.0/.gitignore

@@ -0,0 +1,93 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.project
+.pydevproject
+.settings/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# ruff
+.ruff_cache/
+
+# Evaldeck specific
+.evaldeck/
+evaldeck_results/
+*.evaldeck.json
+
+# Jupyter
+.ipynb_checkpoints/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Internal
+internal/

evaldeck-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - pydantic>=2.0
+          - click>=8.0
+          - types-PyYAML
+        args: [--ignore-missing-imports]
+        files: ^src/

evaldeck-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,227 @@
+# Contributing to Evaldeck
+
+Thank you for your interest in contributing to Evaldeck! This document provides guidelines and instructions for contributing.
+
+## Code of Conduct
+
+By participating in this project, you agree to maintain a respectful and inclusive environment for everyone.
+
+## How to Contribute
+
+### Reporting Bugs
+
+Before submitting a bug report:
+1. Check existing issues to avoid duplicates
+2. Use the latest version of Evaldeck
+3. Collect relevant information (Python version, OS, stack trace)
+
+When submitting a bug report, include:
+- A clear, descriptive title
+- Steps to reproduce the issue
+- Expected vs actual behavior
+- Code samples if applicable
+- Environment details
+
+### Suggesting Features
+
+Feature requests are welcome! Please:
+1. Check existing issues and discussions first
+2. Describe the use case and problem you're trying to solve
+3. Explain how the feature would work
+4. Consider if it fits Evaldeck's scope (agent evaluation)
+
+### Pull Requests
+
+1. **Fork and clone** the repository
+2. **Create a branch** from `main`:
+   ```bash
+   git checkout -b feature/your-feature-name
+   ```
+3. **Set up development environment**:
+   ```bash
+   pip install -e ".[dev]"
+   pre-commit install
+   ```
+4. **Make your changes** following our coding standards
+5. **Add tests** for new functionality
+6. **Run the test suite**:
+   ```bash
+   pytest
+   ruff check .
+   mypy src/
+   ```
+7. **Commit your changes** with a clear message:
+   ```bash
+   git commit -m "feat: add support for X"
+   ```
+8. **Push and open a PR** against `main`
+
+## Development Setup
+
+### Prerequisites
+
+- Python 3.10+
+- Git
+
+### Installation
+
+```bash
+# Clone your fork
+git clone https://github.com/YOUR_USERNAME/evaldeck.git
+cd evaldeck
+
+# Create virtual environment
+python -m venv venv
+source venv/bin/activate  # or `venv\Scripts\activate` on Windows
+
+# Install in development mode
+pip install -e ".[dev]"
+
+# Install pre-commit hooks
+pre-commit install
+```
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest
+
+# Run with coverage
+pytest --cov=evaldeck
+
+# Run specific test file
+pytest tests/test_evaluator.py
+
+# Run specific test
+pytest tests/test_evaluator.py::test_basic_evaluation
+```
+
+### Code Quality
+
+We use:
+- **Ruff** for linting and formatting
+- **mypy** for type checking
+
+```bash
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Type check
+mypy src/
+```
+
+## Coding Standards
+
+### Style
+
+- Follow PEP 8
+- Use type hints for all public functions
+- Maximum line length: 100 characters
+- Use descriptive variable and function names
+
+### Documentation
+
+- Add docstrings to public functions and classes
+- Update README.md if adding user-facing features
+- Add inline comments for complex logic
+
+### Testing
+
+- Write tests for all new functionality
+- Maintain or improve test coverage
+- Use descriptive test names: `test_evaluator_returns_failure_when_tool_missing`
+
+### Commit Messages
+
+Follow [Conventional Commits](https://www.conventionalcommits.org/):
+
+```
+feat: add CrewAI integration
+fix: handle empty trace gracefully
+docs: update installation instructions
+test: add tests for LLM grader
+refactor: simplify metric calculation
+chore: update dependencies
+```
+
+## Project Structure
+
+```
+evaldeck/
+├── src/evaldeck/
+│   ├── __init__.py        # Public API exports
+│   ├── cli.py             # CLI commands
+│   ├── config.py          # Configuration loading
+│   ├── evaluator.py       # Main evaluation engine
+│   ├── trace.py           # Trace data models
+│   ├── test_case.py       # Test case data models
+│   ├── graders/           # Grader implementations
+│   │   ├── __init__.py
+│   │   ├── base.py
+│   │   ├── code.py
+│   │   └── llm.py
+│   ├── metrics/           # Metric implementations
+│   │   ├── __init__.py
+│   │   └── ...
+│   └── integrations/      # Framework adapters
+│       ├── __init__.py
+│       └── langchain.py
+├── tests/
+│   ├── conftest.py
+│   ├── test_evaluator.py
+│   └── ...
+├── examples/
+│   └── ...
+└── docs/
+    └── ...
+```
+
+## Adding a New Integration
+
+To add support for a new agent framework:
+
+1. Create `src/evaldeck/integrations/your_framework.py`
+2. Implement a tracer/adapter that captures execution into `Trace` format
+3. Add optional dependency to `pyproject.toml`
+4. Add tests in `tests/integrations/test_your_framework.py`
+5. Update README.md with usage example
+6. Add example in `examples/`
+
+## Adding a New Grader
+
+To add a new grader type:
+
+1. Create grader class inheriting from `BaseGrader`
+2. Implement `grade(trace, test_case) -> GradeResult`
+3. Add tests
+4. Export from `evaldeck.graders`
+5. Document in README.md
+
+## Adding a New Metric
+
+To add a new metric:
+
+1. Create metric class inheriting from `BaseMetric`
+2. Implement `calculate(trace, test_case) -> MetricResult`
+3. Add tests
+4. Export from `evaldeck.metrics`
+5. Document in README.md
+
+## Getting Help
+
+- Open a [Discussion](https://github.com/tantra-run/evaldeck-py/discussions) for questions
+- Join our Discord (coming soon)
+- Tag maintainers on complex issues
+
+## Recognition
+
+Contributors will be recognized in:
+- CONTRIBUTORS.md
+- Release notes
+- Project documentation
+
+Thank you for contributing to Evaldeck!