pytest-llm-assert 0.1.0__tar.gz

pytest_llm_assert-0.1.0/.env.example

@@ -0,0 +1,27 @@
+# pytest-llm-assert environment configuration
+# Copy this file to .env and fill in your values
+
+# =============================================================================
+# Azure OpenAI (with Entra ID / Azure AD)
+# =============================================================================
+# pip install pytest-llm-assert[azure]
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_DEPLOYMENT=gpt-5-mini
+
+# =============================================================================
+# Google Vertex AI
+# =============================================================================
+# Run: gcloud auth application-default login
+GCP_PROJECT_ID=your-project-id
+GCP_LOCATION=us-central1
+VERTEX_MODEL=gemini-2.0-flash
+
+# =============================================================================
+# OpenAI (direct)
+# =============================================================================
+# OPENAI_API_KEY=sk-...
+
+# =============================================================================
+# Anthropic
+# =============================================================================
+# ANTHROPIC_API_KEY=sk-ant-...

pytest_llm_assert-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,105 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run ruff linter
+        run: uv run ruff check .
+
+      - name: Run ruff formatter check
+        run: uv run ruff format --check .
+
+  type-check:
+    name: Type Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run pyright
+        run: uv run pyright src/pytest_llm_assert/
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run unit tests
+        run: uv run pytest tests/unit/ -v --tb=short
+
+  test-smoke:
+    name: Smoke Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Verify package imports
+        run: |
+          uv run python -c "from pytest_llm_assert import LLMAssert; print('Import successful')"

pytest_llm_assert-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,110 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v[0-9]*"
+
+jobs:
+  build:
+    name: Build Package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Extract version from tag
+        id: get_version
+        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
+
+      - name: Update version in pyproject.toml
+        run: |
+          sed -i 's/^version = ".*"/version = "${{ steps.get_version.outputs.VERSION }}"/' pyproject.toml
+          echo "Updated version to ${{ steps.get_version.outputs.VERSION }}"
+          grep "^version" pyproject.toml
+
+      - name: Install build dependencies
+        run: uv sync --dev
+
+      - name: Build package
+        run: uv build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  test:
+    name: Test Build
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install package from wheel
+        run: pip install dist/*.whl
+
+      - name: Verify installation
+        run: |
+          python -c "from pytest_llm_assert import LLMAssert; print('Import successful')"
+
+  publish-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build, test]
+    environment:
+      name: pypi
+      url: https://pypi.org/project/pytest-llm-assert/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    runs-on: ubuntu-latest
+    needs: [publish-pypi]
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true
+          draft: false
+          prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') || contains(github.ref, 'rc') }}

pytest_llm_assert-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+*.egg
+*.manifest
+*.spec
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.project
+.pydevproject
+.settings/
+
+# Testing
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.nox/
+.hypothesis/
+.ruff_cache/
+.mypy_cache/
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# OS
+.DS_Store
+Thumbs.db
+*.log
+
+# Jupyter
+.ipynb_checkpoints/

pytest_llm_assert-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stefan Broenner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pytest_llm_assert-0.1.0/PKG-INFO

@@ -0,0 +1,246 @@
+Metadata-Version: 2.4
+Name: pytest-llm-assert
+Version: 0.1.0
+Summary: Simple LLM-powered assertions for any pytest test
+Project-URL: Homepage, https://github.com/sbroenne/pytest-llm-assert
+Project-URL: Documentation, https://github.com/sbroenne/pytest-llm-assert#readme
+Project-URL: Repository, https://github.com/sbroenne/pytest-llm-assert
+Author: Stefan Broenner
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,assertions,llm,pytest,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: azure-identity>=1.15
+Requires-Dist: litellm>=1.55
+Requires-Dist: pytest>=8.0
+Provides-Extra: dev
+Requires-Dist: pyright>=1.1; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: python-dotenv>=1.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# pytest-llm-assert
+
+**Natural language assertions for pytest.**
+
+A pytest plugin that lets you write semantic assertions using LLMs. Stop writing brittle string checks — let an LLM understand what you actually mean.
+
+## The Problem
+
+```python
+# ❌ These all fail even though they mean "success":
+assert "success" in response  # Fails on "Succeeded", "successful", "It worked!"
+assert response == "Operation completed successfully"  # Exact match? Really?
+assert re.match(r"success|succeeded|worked", response, re.I)  # Regex hell
+```
+
+```python
+# You're testing a text-to-SQL agent. How do you validate the output?
+
+# ❌ Exact match? There are many valid ways to write the same query:
+assert sql == "SELECT name FROM users WHERE age > 21"
+
+# ❌ Regex? Good luck covering all valid SQL syntax:
+assert re.match(r"SELECT\s+name\s+FROM\s+users", sql, re.I)
+
+# ❌ Parse it? Now you need a SQL parser as a test dependency:
+assert sqlparse.parse(sql)[0].get_type() == "SELECT"
+```
+
+## The Solution
+
+```python
+# ✅ Just say what you mean:
+assert llm(response, "Does this indicate the operation succeeded?")
+assert llm(sql, "Is this a valid SELECT query that returns user names for users over 21?")
+```
+
+## Why This Works
+
+The LLM evaluates your criterion against the content and returns a judgment. It understands:
+
+- **Synonyms**: "success", "succeeded", "worked", "completed" all mean the same thing
+- **Semantics**: Two SQL queries can be equivalent even with different syntax
+- **Context**: "The operation failed successfully" is actually a failure
+- **Intent**: Generated code can be correct even if it's not identical to a reference
+
+
+## Installation
+
+```bash
+pip install pytest-llm-assert
+```
+
+## Setup
+
+This library uses [LiteLLM](https://docs.litellm.ai/) under the hood, giving you access to **100+ LLM providers** with a unified API. 
+
+```bash
+# OpenAI
+export OPENAI_API_KEY=sk-...
+
+# Azure OpenAI with Entra ID (no API keys)
+export AZURE_API_BASE=https://your-resource.openai.azure.com
+export AZURE_API_VERSION=2024-02-15-preview
+# Uses DefaultAzureCredential: az login, managed identity, etc.
+
+# Ollama (local)
+# Just run: ollama serve
+```
+
+See [LiteLLM docs](https://docs.litellm.ai/docs/providers) for all providers including Vertex AI, Bedrock, Anthropic, and more.
+
+## Quick Start
+
+```python
+from pytest_llm_assert import LLMAssert
+
+llm = LLMAssert(model="openai/gpt-5-mini")  # Uses OPENAI_API_KEY from env
+
+# Semantic assertions - returns True/False
+assert llm("Operation completed successfully", "Does this indicate success?")
+assert llm("Error: connection refused", "Does this indicate a failure?")
+assert not llm("All tests passed", "Does this indicate a failure?")
+```
+
+## Real Examples
+
+First, create a fixture in `conftest.py`:
+
+```python
+# conftest.py
+import pytest
+from pytest_llm_assert import LLMAssert
+
+@pytest.fixture
+def llm():
+    return LLMAssert(model="openai/gpt-5-mini")
+```
+
+Then use it in your tests:
+
+### Testing Error Messages
+
+```python
+def test_validation_error_is_helpful(llm):
+    """Error messages should explain the problem clearly."""
+    error_msg = "ValidationError: 'port' must be an integer, got 'not-a-number'"
+    
+    assert llm(error_msg, "Does this explain that port must be a number?")
+    assert llm(error_msg, "Does this indicate which field failed validation?")
+```
+
+### Testing Generated SQL
+
+```python
+def test_query_builder_generates_valid_sql(llm):
+    """Query builder should produce semantically correct SQL."""
+    query = "SELECT name FROM users WHERE age > 21 ORDER BY name"
+    
+    assert llm(query, "Is this a valid SELECT query that returns names of users over 21?")
+```
+
+### Testing LLM Output
+
+```python
+def test_summary_is_comprehensive(llm):
+    """Generated summaries should capture key points."""
+    summary = "The contract establishes a 2-year service agreement between..."
+    
+    assert llm(summary, "Does this summarize a legal contract?")
+    assert llm(summary, "Does this mention the contract duration?")
+```
+
+## Comparing Judge Models
+
+Not sure which LLM to use as your assertion judge? Run the same tests against multiple models to find the best one for your use case:
+
+```python
+import pytest
+from pytest_llm_assert import LLMAssert
+
+MODELS = ["openai/gpt-5-mini", "anthropic/claude-sonnet-4-20250514", "ollama/llama3.1:8b"]
+
+@pytest.fixture(params=MODELS)
+def llm(request):
+    return LLMAssert(model=request.param)
+
+def test_validates_sql_equivalence(llm):
+    """Test which models can judge SQL semantic equivalence."""
+    sql = "SELECT u.name FROM users AS u WHERE u.age >= 22"
+    assert llm(sql, "Is this equivalent to selecting names of users over 21?")
+```
+
+Output shows which judge models correctly evaluate your criterion:
+```
+test_validates_sql_equivalence[openai/gpt-5-mini] PASSED
+test_validates_sql_equivalence[anthropic/claude-sonnet-4-20250514] PASSED  
+test_validates_sql_equivalence[ollama/llama3.1:8b] FAILED
+```
+
+> **Note:** This tests which LLM makes a good *judge* for your assertions. To test AI agents themselves (e.g., "does my coding agent produce working code?"), see [pytest-aitest](https://github.com/sbroenne/pytest-aitest).
+
+## Configuration
+
+### Programmatic
+
+```python
+from pytest_llm_assert import LLMAssert
+
+llm = LLMAssert(
+    model="openai/gpt-5-mini",
+    api_key="sk-...",           # Or use env var
+    api_base="https://...",     # Custom endpoint
+)
+```
+
+### CLI Options
+
+```bash
+pytest --llm-model=openai/gpt-5-mini
+pytest --llm-api-key='${OPENAI_API_KEY}'  # Env var expansion
+pytest --llm-api-base=http://localhost:8080
+```
+
+### Environment Variables
+
+```bash
+export OPENAI_API_KEY=sk-...
+export LLM_MODEL=openai/gpt-5-mini
+```
+
+## API Reference
+
+### `LLMAssert(model, api_key=None, api_base=None, **kwargs)`
+
+Create an LLM assertion helper.
+
+- `model`: LiteLLM model string (e.g., `"openai/gpt-5-mini"`, `"azure/gpt-4o"`)
+- `api_key`: Optional API key (or use environment variables)
+- `api_base`: Optional custom endpoint
+- `**kwargs`: Additional parameters passed to LiteLLM
+
+### `llm(content, criterion) -> AssertionResult`
+
+Evaluate if content meets the criterion.
+
+- Returns `AssertionResult` which is truthy if criterion is met
+- Access `.reasoning` for the LLM's explanation
+
+## See Also
+
+- **[Examples](examples/)** — Example pytest tests showing basic usage, model comparison, and fixture patterns
+- **[pytest-aitest](https://github.com/sbroenne/pytest-aitest)** — Full framework for testing MCP servers, CLIs, and AI agents. Uses pytest-llm-assert for the judge.
+
+## License
+
+MIT