pytest-llm-assert 0.1.0__tar.gz → 0.2.0__tar.gz

pytest_llm_assert-0.2.0/.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Default owners for everything in the repo
+* @sbroenne
+
+# Python source code
+/src/ @sbroenne
+
+# CI/CD configuration
+/.github/ @sbroenne

pytest_llm_assert-0.2.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Report a bug to help us improve
+title: '[BUG] '
+labels: bug
+assignees: ''
+---
+
+## Bug Description
+A clear description of what the bug is.
+
+## To Reproduce
+Steps to reproduce:
+1. ...
+2. ...
+
+## Expected Behavior
+What you expected to happen.
+
+## Actual Behavior
+What actually happened.
+
+## Environment
+- pytest-llm-assert version: 
+- Python version: 
+- OS: 
+- LLM provider/model: 
+
+## Code Example
+```python
+# Minimal code to reproduce the issue
+```
+
+## Error Output
+```
+# Paste any error messages or tracebacks
+```
+
+## Additional Context
+Any other context about the problem.

pytest_llm_assert-0.2.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Documentation
+    url: https://github.com/sbroenne/pytest-llm-assert#readme
+    about: Check the documentation before opening an issue
+  - name: Discussions
+    url: https://github.com/sbroenne/pytest-llm-assert/discussions
+    about: Ask questions and discuss ideas

pytest_llm_assert-0.2.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: '[FEATURE] '
+labels: enhancement
+assignees: ''
+---
+
+## Problem
+Describe the problem or limitation you're facing.
+
+## Proposed Solution
+Describe what you'd like to happen.
+
+## Alternatives Considered
+Describe any alternative solutions you've considered.
+
+## Use Case
+Explain how this feature would benefit users.
+
+## Additional Context
+Any other context, examples, or screenshots.

pytest_llm_assert-0.2.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,25 @@
+## Description
+
+Brief description of the changes.
+
+## Type of Change
+
+- [ ] Bug fix (non-breaking change that fixes an issue)
+- [ ] New feature (non-breaking change that adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] Documentation update
+
+## Checklist
+
+- [ ] I have run `pre-commit run --all-files` and all checks pass
+- [ ] I have added tests that prove my fix/feature works
+- [ ] I have updated the documentation if needed
+- [ ] My changes generate no new warnings
+
+## Testing
+
+Describe the tests you added or ran.
+
+## Related Issues
+
+Fixes #(issue number)

pytest_llm_assert-0.2.0/.github/dependabot.yml

@@ -0,0 +1,27 @@
+version: 2
+updates:
+  # Python dependencies
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "python"
+    commit-message:
+      prefix: "chore(deps)"
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "github-actions"
+    commit-message:
+      prefix: "chore(deps)"

{pytest_llm_assert-0.1.0 → pytest_llm_assert-0.2.0}/.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches: [main, 'feature/**', 'fix/**', 'dev/**']
   pull_request:
     branches: [main]
   workflow_dispatch:
@@ -12,15 +12,15 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
 
@@ -37,15 +37,15 @@ jobs:
     name: Type Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
 
@@ -63,15 +63,15 @@ jobs:
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -85,15 +85,15 @@ jobs:
     name: Smoke Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
 

pytest_llm_assert-0.2.0/.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 00:00 UTC
+    - cron: "0 0 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [python]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+          queries: security-extended,security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v4
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:${{ matrix.language }}"

pytest_llm_assert-0.2.0/.github/workflows/dependency-review.yml

@@ -0,0 +1,23 @@
+name: Dependency Review
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  dependency-review:
+    name: Dependency Review
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: high
+          comment-summary-in-pr: always

{pytest_llm_assert-0.1.0 → pytest_llm_assert-0.2.0}/.github/workflows/release.yml

@@ -10,15 +10,15 @@ jobs:
     name: Build Package
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
 
@@ -50,13 +50,13 @@ jobs:
     needs: build
     steps:
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: dist
           path: dist/
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
 
@@ -78,7 +78,7 @@ jobs:
       id-token: write
     steps:
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: dist
           path: dist/
@@ -93,10 +93,10 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: dist
           path: dist/

pytest_llm_assert-0.2.0/.github/workflows/stale.yml

@@ -0,0 +1,35 @@
+name: Stale Issues and PRs
+
+on:
+  schedule:
+    - cron: "0 0 * * *"  # Run daily at midnight UTC
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v10
+        with:
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not had
+            recent activity. It will be closed if no further activity occurs within 7 days.
+            Thank you for your contributions!
+          stale-pr-message: >
+            This pull request has been automatically marked as stale because it has not had
+            recent activity. It will be closed if no further activity occurs within 7 days.
+          close-issue-message: >
+            This issue was closed because it has been stale for 7 days with no activity.
+            Feel free to reopen if this is still relevant.
+          close-pr-message: >
+            This pull request was closed because it has been stale for 7 days with no activity.
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-label: "stale"
+          stale-pr-label: "stale"
+          exempt-issue-labels: "pinned,security,bug"
+          exempt-pr-labels: "pinned,security"

pytest_llm_assert-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: pyright
+        name: pyright
+        entry: pyright
+        language: system
+        types: [python]
+        pass_filenames: false

pytest_llm_assert-0.2.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,30 @@
+# Code of Conduct
+
+## Our Pledge
+
+We pledge to make participation in our project a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior:
+
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information without explicit permission
+- Other conduct which could reasonably be considered inappropriate
+
+## Enforcement
+
+Project maintainers are responsible for clarifying standards of acceptable behavior and will take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.

pytest_llm_assert-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,116 @@
+# Contributing to pytest-llm-assert
+
+## Development Setup
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/sbroenne/pytest-llm-assert.git
+   cd pytest-llm-assert
+   ```
+
+2. Create a virtual environment and install dependencies:
+   ```bash
+   python -m venv .venv
+   source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+   pip install -e ".[dev]"
+   ```
+
+3. Install pre-commit hooks:
+   ```bash
+   pre-commit install
+   ```
+
+## Code Quality
+
+This project uses automated tools to maintain code quality:
+
+- **[Ruff](https://docs.astral.sh/ruff/)** — Linting and formatting
+- **[Pyright](https://github.com/microsoft/pyright)** — Type checking
+- **[pre-commit](https://pre-commit.com/)** — Git hooks for automated checks
+
+### Running Checks Manually
+
+```bash
+# Run all pre-commit hooks
+pre-commit run --all-files
+
+# Or run individual tools
+ruff check .                 # Linting
+ruff format .                # Formatting
+pyright                      # Type checking
+```
+
+### Pre-commit Hooks
+
+Pre-commit hooks run automatically on `git commit`. If a hook fails, fix the issues and commit again.
+
+The hooks will:
+1. **ruff** — Auto-fix linting issues where possible
+2. **ruff-format** — Format code consistently
+3. **pyright** — Check types
+
+## Running Tests
+
+```bash
+# Run unit tests
+pytest tests/unit/ -v
+
+# Run all tests (requires LLM credentials)
+pytest -v
+
+# Run integration tests only
+pytest -m integration -v
+```
+
+### Integration Tests
+
+Integration tests require LLM provider credentials. Set up at least one:
+
+```bash
+# Azure OpenAI (uses Entra ID - no API key needed)
+export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com
+az login
+
+# Google Vertex AI
+export GOOGLE_CLOUD_PROJECT=your-project-id
+gcloud auth application-default login
+```
+
+## Project Structure
+
+```
+pytest-llm-assert/
+├── src/pytest_llm_assert/
+│   ├── __init__.py      # Package exports
+│   ├── core.py          # LLMAssert implementation
+│   └── plugin.py        # pytest plugin (fixtures, CLI options)
+├── tests/
+│   ├── unit/            # Unit tests (mocked LLM calls)
+│   └── integration/     # Integration tests (real LLM calls)
+├── examples/            # Example usage patterns
+├── pyproject.toml       # Project configuration
+└── .pre-commit-config.yaml
+```
+
+## Making Changes
+
+1. Create a branch for your changes
+2. Make your changes
+3. Ensure all checks pass: `pre-commit run --all-files`
+4. Run tests: `pytest tests/unit/ -v`
+5. Submit a pull request
+
+All PRs are **squash merged** to keep a clean commit history on main.
+
+## Code Style
+
+- Follow [PEP 8](https://peps.python.org/pep-0008/) (enforced by Ruff)
+- Use type hints for all public APIs
+- Keep functions focused and small
+- Write docstrings for public classes and methods
+
+## Releasing
+
+1. Update version in `src/pytest_llm_assert/__init__.py` and `pyproject.toml`
+2. Create a git tag: `git tag v0.x.x`
+3. Push: `git push origin main --tags`

pytest_llm_assert-0.2.0/PKG-INFO

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: pytest-llm-assert
+Version: 0.2.0
+Summary: Simple LLM-powered assertions for any pytest test
+Project-URL: Homepage, https://github.com/sbroenne/pytest-llm-assert
+Project-URL: Documentation, https://github.com/sbroenne/pytest-llm-assert#readme
+Project-URL: Repository, https://github.com/sbroenne/pytest-llm-assert
+Author: Stefan Broenner
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,assertions,llm,pytest,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: azure-identity>=1.25
+Requires-Dist: litellm>=1.81
+Requires-Dist: pytest>=9.0
+Provides-Extra: dev
+Requires-Dist: pre-commit>=4.5; extra == 'dev'
+Requires-Dist: pyright>=1.1.408; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0; extra == 'dev'
+Requires-Dist: pytest>=9.0; extra == 'dev'
+Requires-Dist: python-dotenv>=1.2; extra == 'dev'
+Requires-Dist: ruff>=0.14; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# pytest-llm-assert
+
+[![PyPI version](https://img.shields.io/pypi/v/pytest-llm-assert)](https://pypi.org/project/pytest-llm-assert/)
+[![Python versions](https://img.shields.io/pypi/pyversions/pytest-llm-assert)](https://pypi.org/project/pytest-llm-assert/)
+[![CI](https://github.com/sbroenne/pytest-llm-assert/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-llm-assert/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Natural language assertions for pytest.**
+
+Testing a text-to-SQL agent? Validating LLM-generated code? Checking if error messages are helpful? Now you can:
+
+```python
+def test_sql_agent_output(llm):
+    sql = my_agent.generate("Get names of users over 21")
+    
+    assert llm(sql, "Is this a valid SQL query that selects user names filtered by age > 21?")
+```
+
+The LLM evaluates your criterion and returns pass/fail — no regex, no parsing, no exact string matching.
+
+## Features
+
+- **Semantic assertions** — Assert meaning, not exact strings
+- **100+ LLM providers** — OpenAI, Azure, Anthropic, Ollama, Vertex AI, Bedrock via [LiteLLM](https://docs.litellm.ai/)
+- **pytest native** — Works as a standard pytest plugin/fixture
+- **Response introspection** — Access tokens, cost, and reasoning via `llm.response`
+
+## Installation
+
+```bash
+pip install pytest-llm-assert
+```
+
+## Quick Start
+
+```python
+# conftest.py
+import pytest
+from pytest_llm_assert import LLMAssert
+
+@pytest.fixture
+def llm():
+    return LLMAssert(model="openai/gpt-5-mini")
+```
+
+```python
+# test_my_agent.py
+def test_generated_sql_is_correct(llm):
+    sql = "SELECT name FROM users WHERE age > 21 ORDER BY name"
+    assert llm(sql, "Is this a valid SELECT query that returns names of users over 21?")
+
+def test_error_message_is_helpful(llm):
+    error = "ValidationError: 'port' must be an integer, got 'abc'"
+    assert llm(error, "Does this explain what went wrong and how to fix it?")
+
+def test_summary_captures_key_points(llm):
+    summary = generate_summary(document)
+    assert llm(summary, "Does this mention the contract duration and parties involved?")
+```
+
+## Setup
+
+Works out of the box with cloud identity — no API keys to manage:
+
+```bash
+# Azure (Entra ID)
+export AZURE_API_BASE=https://your-resource.openai.azure.com
+az login
+
+# Google Cloud (Vertex AI)
+gcloud auth application-default login
+
+# AWS (Bedrock)
+aws configure  # Uses IAM credentials
+```
+
+Supports 100+ providers via [LiteLLM](https://docs.litellm.ai/docs/providers) — including API key auth for OpenAI, Anthropic, Ollama, and more.
+
+## Documentation
+
+- **[Configuration](docs/configuration.md)** — All providers, CLI options, environment variables
+- **[API Reference](docs/api-reference.md)** — Full API documentation
+- **[Comparing Judge Models](docs/comparing-models.md)** — Evaluate which LLM works best for your assertions
+- **[Examples](examples/)** — Working pytest examples
+
+## Related
+
+- **[pytest-aitest](https://github.com/sbroenne/pytest-aitest)** — Full framework for testing MCP servers, CLIs, and AI agents
+- **[Contributing](CONTRIBUTING.md)** — Development setup and guidelines
+
+## Requirements
+
+- Python 3.11+
+- pytest 8.0+
+- An LLM (OpenAI, Azure, Anthropic, etc.) or local [Ollama](https://ollama.ai/)
+
+## Security
+
+- **Sensitive data**: Test content is sent to LLM providers — consider data policies
+
+## License
+
+MIT