pytest-llm-assert 0.1.0__py3-none-any.whl

pytest_llm_assert/__init__.py

@@ -0,0 +1,6 @@
+"""pytest-llm-assert: Simple LLM-powered assertions for any pytest test."""
+
+from pytest_llm_assert.core import LLMAssert
+
+__all__ = ["LLMAssert"]
+__version__ = "0.1.0"

pytest_llm_assert/core.py

@@ -0,0 +1,186 @@
+"""Core LLM assertion implementation."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable
+
+import litellm
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@dataclass(slots=True)
+class AssertionResult:
+    """Result of an LLM assertion with rich repr for pytest."""
+
+    passed: bool
+    criterion: str
+    reasoning: str
+    content_preview: str
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+    def __repr__(self) -> str:
+        status = "PASS" if self.passed else "FAIL"
+        return (
+            f"LLMAssert({status}: {self.criterion!r})\n"
+            f"  Content: {self.content_preview!r}\n"
+            f"  Reasoning: {self.reasoning}"
+        )
+
+
+class LLMAssert:
+    """LLM-powered assertions for semantic evaluation.
+
+    Example:
+        >>> llm = LLMAssert(model="openai/gpt-5-mini")
+        >>> assert llm("Hello world", "Is this a greeting?")
+
+    For Azure OpenAI with Entra ID, just use `az login` - no API key needed:
+        >>> llm = LLMAssert(model="azure/gpt-4o", api_base="https://your-resource.openai.azure.com")
+    """
+
+    def __init__(
+        self,
+        model: str = "openai/gpt-5-mini",
+        api_key: str | None = None,
+        api_base: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize LLM assertion helper.
+
+        Args:
+            model: LiteLLM model string (e.g., "openai/gpt-5-mini", "azure/gpt-4o")
+            api_key: API key (supports ${ENV_VAR} expansion). For Azure, leave empty to use Entra ID.
+            api_base: Custom API base URL (required for Azure)
+            **kwargs: Additional parameters passed to LiteLLM
+        """
+        self.model = model
+        self.api_key = self._expand_env(api_key) if api_key else None
+        self.api_base = api_base
+        self.kwargs = kwargs
+        self._azure_ad_token_provider: Callable[[], str] | None = None
+
+        # Auto-configure Azure Entra ID when no API key is provided
+        if self._is_azure_model() and not self._has_azure_api_key():
+            self._azure_ad_token_provider = self._get_azure_ad_token_provider()
+
+    def _is_azure_model(self) -> bool:
+        """Check if the model is an Azure OpenAI model."""
+        return self.model.startswith("azure/")
+
+    def _has_azure_api_key(self) -> bool:
+        """Check if an Azure API key is available."""
+        return bool(self.api_key or os.environ.get("AZURE_API_KEY"))
+
+    @staticmethod
+    def _get_azure_ad_token_provider() -> Callable[[], str] | None:
+        """Get Azure AD token provider for Entra ID authentication.
+
+        Uses LiteLLM's built-in helper which leverages DefaultAzureCredential:
+        - Azure CLI credentials (az login)
+        - Managed Identity
+        - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID)
+        - Visual Studio Code credentials
+        """
+        try:
+            from litellm.secret_managers.get_azure_ad_token_provider import (
+                get_azure_ad_token_provider,
+            )
+
+            return get_azure_ad_token_provider()
+        except ImportError:
+            # azure-identity not installed
+            return None
+        except Exception:
+            # Credential not available
+            return None
+
+    @staticmethod
+    def _expand_env(value: str) -> str:
+        """Expand ${VAR} patterns in string."""
+        pattern = r"\$\{([^}]+)\}"
+        return re.sub(pattern, lambda m: os.environ.get(m.group(1), m.group(0)), value)
+
+    @staticmethod
+    def _truncate(text: str, max_len: int = 100) -> str:
+        """Truncate text for display."""
+        if len(text) <= max_len:
+            return text
+        return text[: max_len - 3] + "..."
+
+    def _call_llm(self, messages: list[dict[str, str]]) -> str:
+        """Call the LLM and return response content."""
+        kwargs = {**self.kwargs}
+
+        # Use Azure AD token provider if configured (Entra ID auth)
+        if self._azure_ad_token_provider is not None:
+            kwargs["azure_ad_token_provider"] = self._azure_ad_token_provider
+
+        response = litellm.completion(
+            model=self.model,
+            messages=messages,
+            api_key=self.api_key,
+            api_base=self.api_base,
+            **kwargs,
+        )
+        return response.choices[0].message.content or ""  # type: ignore[union-attr]
+
+    def __call__(self, content: str, criterion: str) -> AssertionResult:
+        """Evaluate if content meets the given criterion.
+
+        Args:
+            content: The text to evaluate
+            criterion: Plain English criterion (e.g., "Is this professional?")
+
+        Returns:
+            AssertionResult that is truthy if criterion is met
+        """
+        messages = [
+            {
+                "role": "system",
+                "content": (
+                    "You are an assertion evaluator. "
+                    "Evaluate if the given content meets the specified criterion.\n\n"
+                    "Respond in JSON format:\n"
+                    '{"result": "PASS" or "FAIL", "reasoning": "brief explanation"}'
+                ),
+            },
+            {
+                "role": "user",
+                "content": f"Criterion: {criterion}\n\nContent:\n{content}",
+            },
+        ]
+
+        response = self._call_llm(messages)
+
+        # Parse JSON response
+        try:
+            # Handle potential markdown code blocks
+            text = response.strip()
+            if text.startswith("```"):
+                text = text.split("```")[1]
+                if text.startswith("json"):
+                    text = text[4:]
+            data = json.loads(text.strip())
+            passed = data.get("result", "").upper() == "PASS"
+            reasoning = data.get("reasoning", "")
+        except (json.JSONDecodeError, AttributeError):
+            # Fallback to line-based parsing
+            lines = response.strip().split("\n", 1)
+            first_line = lines[0].strip().upper()
+            passed = first_line in ("PASS", "YES", "TRUE", "PASSED")
+            reasoning = lines[1].strip() if len(lines) > 1 else response
+
+        return AssertionResult(
+            passed=passed,
+            criterion=criterion,
+            reasoning=reasoning,
+            content_preview=self._truncate(content),
+        )

pytest_llm_assert/plugin.py

@@ -0,0 +1,43 @@
+"""pytest plugin providing LLM-powered assertion fixtures."""
+
+from __future__ import annotations
+
+import pytest
+
+from pytest_llm_assert.core import LLMAssert
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Add pytest CLI options for LLM assertions."""
+    group = parser.getgroup("llm-assert", "LLM-powered assertions")
+    group.addoption(
+        "--llm-model",
+        default="openai/gpt-5-mini",
+        help="Default LiteLLM model for assertions (default: openai/gpt-5-mini)",
+    )
+    group.addoption(
+        "--llm-api-key",
+        default=None,
+        help="API key for LLM provider (supports ${ENV_VAR} expansion)",
+    )
+    group.addoption(
+        "--llm-api-base",
+        default=None,
+        help="Custom API base URL for LLM provider",
+    )
+
+
+@pytest.fixture
+def llm_assert(request: pytest.FixtureRequest) -> LLMAssert:
+    """Fixture providing an LLMAssert instance configured via CLI options.
+
+    Example:
+        def test_greeting(llm_assert):
+            response = "Hello! How can I help you today?"
+            assert llm_assert(response, "Is this a friendly greeting?")
+    """
+    return LLMAssert(
+        model=request.config.getoption("--llm-model"),
+        api_key=request.config.getoption("--llm-api-key"),
+        api_base=request.config.getoption("--llm-api-base"),
+    )

pytest_llm_assert-0.1.0.dist-info/METADATA

@@ -0,0 +1,246 @@
+Metadata-Version: 2.4
+Name: pytest-llm-assert
+Version: 0.1.0
+Summary: Simple LLM-powered assertions for any pytest test
+Project-URL: Homepage, https://github.com/sbroenne/pytest-llm-assert
+Project-URL: Documentation, https://github.com/sbroenne/pytest-llm-assert#readme
+Project-URL: Repository, https://github.com/sbroenne/pytest-llm-assert
+Author: Stefan Broenner
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,assertions,llm,pytest,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: azure-identity>=1.15
+Requires-Dist: litellm>=1.55
+Requires-Dist: pytest>=8.0
+Provides-Extra: dev
+Requires-Dist: pyright>=1.1; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: python-dotenv>=1.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# pytest-llm-assert
+
+**Natural language assertions for pytest.**
+
+A pytest plugin that lets you write semantic assertions using LLMs. Stop writing brittle string checks — let an LLM understand what you actually mean.
+
+## The Problem
+
+```python
+# ❌ These all fail even though they mean "success":
+assert "success" in response  # Fails on "Succeeded", "successful", "It worked!"
+assert response == "Operation completed successfully"  # Exact match? Really?
+assert re.match(r"success|succeeded|worked", response, re.I)  # Regex hell
+```
+
+```python
+# You're testing a text-to-SQL agent. How do you validate the output?
+
+# ❌ Exact match? There are many valid ways to write the same query:
+assert sql == "SELECT name FROM users WHERE age > 21"
+
+# ❌ Regex? Good luck covering all valid SQL syntax:
+assert re.match(r"SELECT\s+name\s+FROM\s+users", sql, re.I)
+
+# ❌ Parse it? Now you need a SQL parser as a test dependency:
+assert sqlparse.parse(sql)[0].get_type() == "SELECT"
+```
+
+## The Solution
+
+```python
+# ✅ Just say what you mean:
+assert llm(response, "Does this indicate the operation succeeded?")
+assert llm(sql, "Is this a valid SELECT query that returns user names for users over 21?")
+```
+
+## Why This Works
+
+The LLM evaluates your criterion against the content and returns a judgment. It understands:
+
+- **Synonyms**: "success", "succeeded", "worked", "completed" all mean the same thing
+- **Semantics**: Two SQL queries can be equivalent even with different syntax
+- **Context**: "The operation failed successfully" is actually a failure
+- **Intent**: Generated code can be correct even if it's not identical to a reference
+
+
+## Installation
+
+```bash
+pip install pytest-llm-assert
+```
+
+## Setup
+
+This library uses [LiteLLM](https://docs.litellm.ai/) under the hood, giving you access to **100+ LLM providers** with a unified API. 
+
+```bash
+# OpenAI
+export OPENAI_API_KEY=sk-...
+
+# Azure OpenAI with Entra ID (no API keys)
+export AZURE_API_BASE=https://your-resource.openai.azure.com
+export AZURE_API_VERSION=2024-02-15-preview
+# Uses DefaultAzureCredential: az login, managed identity, etc.
+
+# Ollama (local)
+# Just run: ollama serve
+```
+
+See [LiteLLM docs](https://docs.litellm.ai/docs/providers) for all providers including Vertex AI, Bedrock, Anthropic, and more.
+
+## Quick Start
+
+```python
+from pytest_llm_assert import LLMAssert
+
+llm = LLMAssert(model="openai/gpt-5-mini")  # Uses OPENAI_API_KEY from env
+
+# Semantic assertions - returns True/False
+assert llm("Operation completed successfully", "Does this indicate success?")
+assert llm("Error: connection refused", "Does this indicate a failure?")
+assert not llm("All tests passed", "Does this indicate a failure?")
+```
+
+## Real Examples
+
+First, create a fixture in `conftest.py`:
+
+```python
+# conftest.py
+import pytest
+from pytest_llm_assert import LLMAssert
+
+@pytest.fixture
+def llm():
+    return LLMAssert(model="openai/gpt-5-mini")
+```
+
+Then use it in your tests:
+
+### Testing Error Messages
+
+```python
+def test_validation_error_is_helpful(llm):
+    """Error messages should explain the problem clearly."""
+    error_msg = "ValidationError: 'port' must be an integer, got 'not-a-number'"
+    
+    assert llm(error_msg, "Does this explain that port must be a number?")
+    assert llm(error_msg, "Does this indicate which field failed validation?")
+```
+
+### Testing Generated SQL
+
+```python
+def test_query_builder_generates_valid_sql(llm):
+    """Query builder should produce semantically correct SQL."""
+    query = "SELECT name FROM users WHERE age > 21 ORDER BY name"
+    
+    assert llm(query, "Is this a valid SELECT query that returns names of users over 21?")
+```
+
+### Testing LLM Output
+
+```python
+def test_summary_is_comprehensive(llm):
+    """Generated summaries should capture key points."""
+    summary = "The contract establishes a 2-year service agreement between..."
+    
+    assert llm(summary, "Does this summarize a legal contract?")
+    assert llm(summary, "Does this mention the contract duration?")
+```
+
+## Comparing Judge Models
+
+Not sure which LLM to use as your assertion judge? Run the same tests against multiple models to find the best one for your use case:
+
+```python
+import pytest
+from pytest_llm_assert import LLMAssert
+
+MODELS = ["openai/gpt-5-mini", "anthropic/claude-sonnet-4-20250514", "ollama/llama3.1:8b"]
+
+@pytest.fixture(params=MODELS)
+def llm(request):
+    return LLMAssert(model=request.param)
+
+def test_validates_sql_equivalence(llm):
+    """Test which models can judge SQL semantic equivalence."""
+    sql = "SELECT u.name FROM users AS u WHERE u.age >= 22"
+    assert llm(sql, "Is this equivalent to selecting names of users over 21?")
+```
+
+Output shows which judge models correctly evaluate your criterion:
+```
+test_validates_sql_equivalence[openai/gpt-5-mini] PASSED
+test_validates_sql_equivalence[anthropic/claude-sonnet-4-20250514] PASSED  
+test_validates_sql_equivalence[ollama/llama3.1:8b] FAILED
+```
+
+> **Note:** This tests which LLM makes a good *judge* for your assertions. To test AI agents themselves (e.g., "does my coding agent produce working code?"), see [pytest-aitest](https://github.com/sbroenne/pytest-aitest).
+
+## Configuration
+
+### Programmatic
+
+```python
+from pytest_llm_assert import LLMAssert
+
+llm = LLMAssert(
+    model="openai/gpt-5-mini",
+    api_key="sk-...",           # Or use env var
+    api_base="https://...",     # Custom endpoint
+)
+```
+
+### CLI Options
+
+```bash
+pytest --llm-model=openai/gpt-5-mini
+pytest --llm-api-key='${OPENAI_API_KEY}'  # Env var expansion
+pytest --llm-api-base=http://localhost:8080
+```
+
+### Environment Variables
+
+```bash
+export OPENAI_API_KEY=sk-...
+export LLM_MODEL=openai/gpt-5-mini
+```
+
+## API Reference
+
+### `LLMAssert(model, api_key=None, api_base=None, **kwargs)`
+
+Create an LLM assertion helper.
+
+- `model`: LiteLLM model string (e.g., `"openai/gpt-5-mini"`, `"azure/gpt-4o"`)
+- `api_key`: Optional API key (or use environment variables)
+- `api_base`: Optional custom endpoint
+- `**kwargs`: Additional parameters passed to LiteLLM
+
+### `llm(content, criterion) -> AssertionResult`
+
+Evaluate if content meets the criterion.
+
+- Returns `AssertionResult` which is truthy if criterion is met
+- Access `.reasoning` for the LLM's explanation
+
+## See Also
+
+- **[Examples](examples/)** — Example pytest tests showing basic usage, model comparison, and fixture patterns
+- **[pytest-aitest](https://github.com/sbroenne/pytest-aitest)** — Full framework for testing MCP servers, CLIs, and AI agents. Uses pytest-llm-assert for the judge.
+
+## License
+
+MIT

pytest_llm_assert-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+pytest_llm_assert/__init__.py,sha256=gp_z4g6Yf9SnjwEyZc6kPSqEWw2Nyb5er84HRuUaXCA,169
+pytest_llm_assert/core.py,sha256=sDQvcus5EqHQ-_iQyLH2XB9nL4UhLpiWGTXnGhO7YyE,6351
+pytest_llm_assert/plugin.py,sha256=g3sotHAeUXMuOsFQdaoIbn0CY24i-1CPv0EglrC5qtE,1327
+pytest_llm_assert-0.1.0.dist-info/METADATA,sha256=cGK3fmb5T0ZKOBtM0PkmnRkAaGnLZ1aEDhBD5U8-1UQ,7713
+pytest_llm_assert-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+pytest_llm_assert-0.1.0.dist-info/entry_points.txt,sha256=YEYg83TT6znVYdvFvZHJEOJ8XsZbcrqV9pY8uM-ThQE,49
+pytest_llm_assert-0.1.0.dist-info/licenses/LICENSE,sha256=wHrdHpzRm4rdlyMdj-sQw7aou6kHPujW0VmRBEhInJ8,1072
+pytest_llm_assert-0.1.0.dist-info/RECORD,,

pytest_llm_assert-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_llm_assert-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+llm_assert = pytest_llm_assert.plugin

pytest_llm_assert-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stefan Broenner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.