agentprobe-framework 1.0.0__tar.gz

agentprobe_framework-1.0.0/.env.example

@@ -0,0 +1,9 @@
+# AgentProbe Environment Variables
+# Copy this file to .env and fill in your values.
+
+# Provider API keys (for agents that call these providers)
+# ANTHROPIC_API_KEY=your-key-here
+# OPENAI_API_KEY=your-key-here
+
+# PostgreSQL connection (optional, for team storage backend)
+# AGENTPROBE_PG_DSN=postgresql://user:pass@localhost:5432/agentprobe

agentprobe_framework-1.0.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,26 @@
+---
+name: Bug Report
+about: Report a bug in AgentProbe
+title: "[BUG] "
+labels: bug
+assignees: ""
+---
+
+## Description
+<!-- A clear description of the bug. -->
+
+## Steps to Reproduce
+1.
+2.
+3.
+
+## Expected Behavior
+<!-- What you expected to happen. -->
+
+## Actual Behavior
+<!-- What actually happened. -->
+
+## Environment
+- AgentProbe version:
+- Python version:
+- OS:

agentprobe_framework-1.0.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,19 @@
+---
+name: Feature Request
+about: Suggest a new feature for AgentProbe
+title: "[FEATURE] "
+labels: enhancement
+assignees: ""
+---
+
+## Problem
+<!-- What problem does this feature solve? -->
+
+## Proposed Solution
+<!-- How should it work? -->
+
+## Alternatives Considered
+<!-- Any alternative approaches you've considered. -->
+
+## Additional Context
+<!-- Any other context, screenshots, or examples. -->

agentprobe_framework-1.0.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+## Summary
+<!-- What does this PR do? 1-3 sentences. -->
+
+## Related Issues
+<!-- Closes #123 -->
+
+## Changes
+<!-- List of significant changes -->
+
+## Testing
+<!-- How was this tested? -->
+- [ ] Unit tests added/updated
+- [ ] Integration tests added/updated (if applicable)
+- [ ] Manual testing performed
+
+## Checklist
+- [ ] Code is type-annotated (mypy strict passes)
+- [ ] Docstrings added for all public APIs
+- [ ] Tests pass with >90% coverage for new code
+- [ ] No linting errors (ruff)
+- [ ] Changelog entry added (if user-facing)
+- [ ] Documentation updated (if applicable)

agentprobe_framework-1.0.0/.github/dependabot.yml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    target-branch: "develop"
+    open-pull-requests-limit: 10
+    labels:
+      - "dependencies"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    target-branch: "develop"
+    open-pull-requests-limit: 5
+    labels:
+      - "ci"

agentprobe_framework-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,72 @@
+name: CI
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+      - name: Run ruff check
+        run: ruff check src/ tests/
+      - name: Run ruff format check
+        run: ruff format --check src/ tests/
+
+  type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+      - name: Run mypy
+        run: mypy src/agentprobe/
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: pip install -e ".[dev,test]"
+      - name: Run tests
+        run: |
+          pytest tests/ \
+            --cov=agentprobe \
+            --cov-report=xml:coverage.xml \
+            --cov-report=term-missing \
+            --junitxml=test-results.xml \
+            -v
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.python-version }}
+          path: test-results.xml
+      - name: Upload coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: coverage.xml
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.11'
+        uses: codecov/codecov-action@v4
+        with:
+          file: coverage.xml

agentprobe_framework-1.0.0/.github/workflows/docs.yml

@@ -0,0 +1,23 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: pip install -e ".[docs]"
+      - name: Deploy to GitHub Pages
+        run: mkdocs gh-deploy --force

agentprobe_framework-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: Release to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install build dependencies
+        run: pip install build
+      - name: Build package
+        run: python -m build
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

agentprobe_framework-1.0.0/.gitignore

@@ -0,0 +1,42 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing / Coverage
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+htmlcov/
+.coverage
+coverage.xml
+
+# Environment
+.env
+.env.*
+!.env.example
+
+# OS
+.DS_Store
+Thumbs.db
+
+# AgentProbe runtime
+.agentprobe/
+agentprobe-report/

agentprobe_framework-1.0.0/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-check
+        name: ruff lint
+        entry: ruff check --fix
+        language: system
+        types: [python]
+
+      - id: ruff-format
+        name: ruff format
+        entry: ruff format
+        language: system
+        types: [python]
+
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: system
+        types: [python]
+        args: [--config-file=pyproject.toml]
+

agentprobe_framework-1.0.0/CONTRIBUTING.md

@@ -0,0 +1,181 @@
+# Contributing to AgentProbe
+
+Thank you for your interest in contributing to AgentProbe! This document covers how to set up your development environment, our coding standards, and the contribution workflow.
+
+## Development Setup
+
+### Prerequisites
+
+- Python 3.11 or later
+- Git
+
+### Getting Started
+
+```bash
+# Clone the repository
+git clone https://github.com/dyrach1o/agentprobe-framework.git
+cd agentprobe-framework
+
+# Create a virtual environment
+python -m venv .venv
+source .venv/bin/activate  # or .venv\Scripts\activate on Windows
+
+# Install with dev dependencies
+make dev
+# or manually: pip install -e ".[dev,test,docs]"
+
+# Verify setup
+make check
+```
+
+## Coding Standards
+
+### Python
+
+- **Python 3.11+** required
+- **ruff** for linting and formatting --- run `make lint` and `make format`
+- **mypy strict mode** for type checking --- run `make type-check`
+- **100% type annotations** on all public and private functions
+- **Google-style docstrings** on all public classes, methods, and functions
+- Line length: 100 characters
+
+### Naming Conventions
+
+| Scope | Convention | Example |
+|-------|-----------|---------|
+| Modules | `snake_case` | `trace_compare.py` |
+| Classes | `PascalCase` | `TraceRecorder` |
+| Functions | `snake_case` | `calculate_cost()` |
+| Constants | `SCREAMING_SNAKE_CASE` | `MAX_TIMEOUT_SECONDS` |
+| Variables | `snake_case` | `total_cost` |
+
+### Import Order
+
+Enforced by ruff (isort):
+
+1. Standard library
+2. Third-party packages
+3. First-party (`agentprobe`)
+
+## Testing
+
+### Running Tests
+
+```bash
+# All tests
+make test
+
+# Unit tests only
+make test-unit
+
+# Fast tests (skip slow and API tests)
+make test-fast
+
+# Specific test file
+pytest tests/unit/core/test_runner.py -v
+```
+
+### Writing Tests
+
+- Test files go in `tests/unit/` mirroring the `src/` structure
+- Use pytest fixtures from `tests/fixtures/` for test data
+- Use factory functions (`make_trace`, `make_eval_result`, etc.) for model instances
+- Follow the pattern: `test_{method}_{scenario}_{expected_outcome}`
+- Async tests use `@pytest.mark.asyncio`
+- Minimum **90% coverage** for new code
+
+### Test Structure
+
+```python
+class TestMyClass:
+    """Tests for MyClass."""
+
+    def test_method_with_valid_input_returns_expected(self) -> None:
+        # Arrange
+        obj = MyClass()
+
+        # Act
+        result = obj.method(valid_input)
+
+        # Assert
+        assert result == expected
+
+    def test_method_with_invalid_input_raises_error(self) -> None:
+        obj = MyClass()
+
+        with pytest.raises(ValueError, match="specific message"):
+            obj.method(invalid_input)
+```
+
+## Git Workflow
+
+### Branches
+
+- `main` --- production-ready, protected
+- `develop` --- integration branch
+- `feature/TEAM-X.YY.ZZ-description` --- feature branches from `develop`
+- `bugfix/TEAM-X.YY.ZZ-description` --- bug fixes
+- `hotfix/description` --- urgent fixes from `main`
+
+### Commit Messages
+
+We use [Conventional Commits](https://www.conventionalcommits.org/):
+
+```
+<type>(<scope>): <description>
+```
+
+**Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`
+
+**Scopes:** `core`, `eval`, `safety`, `adapters`, `cli`, `trace`, `cost`, `storage`, `reporting`, `plugins`, `docs`, `infra`
+
+**Examples:**
+
+```
+feat(core): implement test runner with parallel execution
+fix(eval): correct cosine similarity for zero vectors
+docs(cli): add usage examples for trace commands
+test(safety): add prompt injection detection tests
+```
+
+### Pull Request Process
+
+1. Create a feature branch from `develop`
+2. Make your changes following the coding standards
+3. Ensure all checks pass: `make check`
+4. Push and open a PR against `develop`
+5. Fill out the PR template
+6. Address review feedback
+
+### PR Checklist
+
+- [ ] Code is fully type-annotated (mypy strict passes)
+- [ ] Docstrings added for all public APIs
+- [ ] Tests pass with >= 90% coverage for new code
+- [ ] No linting errors (`ruff check`)
+- [ ] Changelog entry added (if user-facing)
+- [ ] Documentation updated (if applicable)
+
+## Project Layout
+
+```
+src/agentprobe/
+├── core/       # Test runner, discovery, assertions, config
+├── eval/       # Evaluators
+├── trace/      # Recording and replay
+├── cost/       # Cost tracking
+├── safety/     # Safety scanning
+├── regression/ # Regression detection
+├── adapters/   # Framework adapters
+├── metrics/    # Metric collection
+├── storage/    # Storage backends
+├── reporting/  # Output formatters
+├── plugins/    # Plugin system
+└── cli/        # CLI commands
+```
+
+## Getting Help
+
+- Open a [GitHub Issue](https://github.com/dyrach1o/agentprobe-framework/issues) for bugs or feature requests
+- Check existing issues before creating new ones
+- Include reproduction steps for bug reports

agentprobe_framework-1.0.0/Dockerfile

@@ -0,0 +1,33 @@
+# ── Stage 1: Builder ──
+FROM python:3.11-slim AS builder
+
+WORKDIR /build
+
+COPY pyproject.toml README.md LICENSE ./
+COPY src/ src/
+
+RUN pip install --no-cache-dir build \
+    && python -m build --wheel --outdir /build/dist
+
+# ── Stage 2: Runtime ──
+FROM python:3.11-slim AS runtime
+
+# Create non-root user
+RUN groupadd --gid 1000 agentprobe \
+    && useradd --uid 1000 --gid 1000 --create-home agentprobe
+
+WORKDIR /app
+
+# Install the wheel from builder stage
+COPY --from=builder /build/dist/*.whl /tmp/
+RUN pip install --no-cache-dir /tmp/*.whl \
+    && rm -rf /tmp/*.whl
+
+# Switch to non-root user
+USER agentprobe
+
+# Default working directory for user projects
+WORKDIR /project
+
+ENTRYPOINT ["agentprobe"]
+CMD ["--help"]