quack-test 0.1.0__tar.gz

quack_test-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Michael F.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quack_test-0.1.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: quack-test
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Dynamic: license-file
+
+# Quack Test
+
+A plugin for pytest to evaluate non-deterministic agent components.
+
+## Installation
+
+Simply pip install it
+
+```bash
+pip install quack-test
+```
+
+## Usage
+
+You can simply specify `@nondeterministic` for components, which are not deterministic and should be run multiple times.
+
+```python
+import random
+from quack_test import nondeterministic_fixture, nondeterministic_test, judge
+
+@nondeterministic_fixture(n=5)
+def sample_text():
+    return f"I have {random.randint(10)} apples."
+
+@nondeterministic_test(score=0.8)
+def test_apples(sample_text):
+    return judge(sample_text, criterion="Has more than 1 apple")
+
+@nondeterministic_test(score=0.8)
+def test_coal(sample_text):
+    return judge(sample_text, criterion="Has only coal")
+```

quack_test-0.1.0/README.md

@@ -0,0 +1,32 @@
+# Quack Test
+
+A plugin for pytest to evaluate non-deterministic agent components.
+
+## Installation
+
+Simply pip install it
+
+```bash
+pip install quack-test
+```
+
+## Usage
+
+You can simply specify `@nondeterministic` for components, which are not deterministic and should be run multiple times.
+
+```python
+import random
+from quack_test import nondeterministic_fixture, nondeterministic_test, judge
+
+@nondeterministic_fixture(n=5)
+def sample_text():
+    return f"I have {random.randint(10)} apples."
+
+@nondeterministic_test(score=0.8)
+def test_apples(sample_text):
+    return judge(sample_text, criterion="Has more than 1 apple")
+
+@nondeterministic_test(score=0.8)
+def test_coal(sample_text):
+    return judge(sample_text, criterion="Has only coal")
+```

quack_test-0.1.0/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "quack-test"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "pytest>=7.0.0",
+    "openai>=1.0.0",
+    "python-dotenv>=1.0.0",
+]
+
+[dependency-groups]
+dev = [
+    "ty",
+    "ruff",
+    "pre-commit",
+]

quack_test-0.1.0/quack_test/__init__.py

@@ -0,0 +1,15 @@
+"""
+Quack Test - Decorators for handling nondeterministic tests and fixtures.
+
+Provides decorators for running fixtures and tests multiple times to handle
+flaky behavior, particularly useful for LLM-based tests.
+"""
+
+from quack_test.decorators import nondeterministic_fixture, nondeterministic_test
+from quack_test.judge import judge
+
+__all__ = [
+    "nondeterministic_fixture",
+    "nondeterministic_test",
+    "judge",
+]

quack_test-0.1.0/quack_test/decorators.py

@@ -0,0 +1,147 @@
+"""
+Decorators for handling nondeterministic tests and fixtures.
+"""
+
+import functools
+import pytest
+from typing import Callable, Any, List
+
+
+def nondeterministic_fixture(n: int = 5):
+    """
+    Decorator for fixtures that should be executed multiple times.
+
+    The fixture function will be executed `n` times, and all results
+    will be collected into a list. This list is then passed to tests
+    that depend on this fixture.
+
+    Args:
+        n: Number of times to execute the fixture (default: 5)
+
+    Example:
+        @nondeterministic_fixture(n=10)
+        def random_data():
+            return random.randint(1, 100)
+
+        # The test will receive a list of 10 random integers
+        def test_values(random_data):
+            assert len(random_data) == 10
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> List[Any]:
+            results = []
+            for _ in range(n):
+                result = func(*args, **kwargs)
+                results.append(result)
+            return results
+
+        # Mark this as a pytest fixture
+        return pytest.fixture(wrapper)
+
+    return decorator
+
+
+def nondeterministic_test(score: float = 0.8, n: int = -1, should_fail: bool = False):
+    """
+    Decorator for tests that should be executed multiple times with a success threshold.
+
+    The test function will be executed multiple times (determined by the length of
+    fixture data if available, or a default number). The test must return a score
+    which will be averaged. If the average score is higher than the given score, you pass.
+
+    Args:
+        score: Minimum average score that must be achieved in order to pass the test.
+        n: Number of times to run the test (if -1, uses fixture length)
+        should_fail: If True, the test passes when the score is BELOW the threshold (default: False)
+
+    Example:
+        @nondeterministic_fixture(n=10)
+        def llm_output():
+            return call_llm("Generate a greeting")
+
+        @nondeterministic_test(score=0.8)
+        def test_greeting(llm_output):
+            return judge(llm_output, criterion="Contains 'hello' or 'hi'")
+
+        @nondeterministic_test(score=0.8, should_fail=True)
+        def test_no_profanity(llm_output):
+            return judge(llm_output, criterion="Contains profanity")
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> None:
+            # n_runs can be either given by n or defined by the length of the fixture data in args or kwargs
+            # use any list found in args or kwargs as fixture data
+            fixture_data = None
+            for arg in args:
+                if isinstance(arg, list):
+                    fixture_data = arg
+                    break
+            for kwarg in kwargs.values():
+                if isinstance(kwarg, list):
+                    fixture_data = kwarg
+                    break
+            if n > 0:
+                n_runs = n
+            elif fixture_data is not None:
+                n_runs = len(fixture_data)
+            else:
+                raise RuntimeError(
+                    "Cannot determine number of runs for nondeterministic_test. "
+                    "Provide a fixture that returns a list or set n explicitly."
+                )
+
+            # Run the test n times
+            successes = 0
+            failures = 0
+            scores = []
+
+            for i in range(n_runs):
+                try:
+                    # If we have fixture data, pass the i-th element
+                    test_args = [
+                        arg[i] if isinstance(arg, list) and len(arg) == n_runs else arg
+                        for arg in args
+                    ]
+                    test_kwargs = {
+                        k: (v[i] if isinstance(v, list) and len(v) == n_runs else v)
+                        for k, v in kwargs.items()
+                    }
+
+                    result = float(func(*test_args, **test_kwargs))
+                    scores.append(result)
+
+                    if result > score:
+                        successes += 1
+                    else:
+                        failures += 1
+                except Exception:
+                    failures += 1
+                    scores.append(0)
+
+            # Calculate success rate
+            success_rate = successes / n_runs
+            achieved_score = sum(scores) / n_runs
+
+            # Assert the success rate meets the threshold
+            if should_fail:
+                # For should_fail tests, we expect the score to be BELOW the threshold
+                assert achieved_score < score, (
+                    f"Test expected to fail but succeeded. "
+                    f"Score: {achieved_score:.2} (required: < {score:.2}), "
+                    f"Success rate: {success_rate:.2%} ({successes}/{n_runs})"
+                )
+            else:
+                # Normal tests expect the score to be AT OR ABOVE the threshold
+                assert achieved_score >= score, (
+                    f"Test failed to meet success threshold. "
+                    f"Score: {achieved_score:.2} (required: {score:.2}), "
+                    f"Success rate: {success_rate:.2%} ({successes}/{n_runs})"
+                )
+
+        return wrapper
+
+    return decorator

quack_test-0.1.0/quack_test/judge.py

@@ -0,0 +1,115 @@
+"""
+Judge function for evaluating test outputs against criteria.
+"""
+import os
+from functools import cache
+from pathlib import Path
+from typing import Optional
+from dotenv import load_dotenv
+from openai import OpenAI, AzureOpenAI
+
+# Load environment variables
+load_dotenv()
+
+# Get the directory where this file is located
+_PROMPTS_DIR = Path(__file__).parent
+
+
+@cache
+def _load_prompt_template(filepath: Path) -> str:
+    """Load a prompt template from a markdown file with caching."""
+    with open(filepath, 'r', encoding='utf-8') as f:
+        return f.read()
+
+
+@cache
+def _get_client():
+    """Create and cache the OpenAI or Azure OpenAI client."""
+    provider = os.getenv("OPENAI_PROVIDER", "OpenAI")
+    
+    if provider == "AzureOpenAI":
+        return AzureOpenAI(
+            api_key=os.environ["OPENAI_API_KEY"],
+            base_url=os.environ["OPENAI_ENDPOINT"],
+            api_version=os.getenv("OPENAI_API_VERSION", "2024-10-21"),
+        )
+    else:
+        return OpenAI(
+            api_key=os.environ["OPENAI_API_KEY"],
+            base_url=os.environ["OPENAI_ENDPOINT"],
+        )
+
+
+def judge(
+    text: str, 
+    criterion: str = "", 
+    gt: str = "", 
+    prompt_template: Optional[str] = None
+) -> float:
+    """
+    Evaluate whether an text meets a given criterion or matches the ground truth.
+    
+    Args:
+        text: The text to evaluate (typically a string)
+        criterion: (Optional) The criterion to check against
+        gt: (Optional) A ground truth which the text should match
+        prompt_template: (Optional) Custom prompt template string with {text}, {criterion}, and/or {gt} placeholders.
+                        If not provided, loads from criterion_prompt.md or gt_prompt.md
+    
+    Returns:
+        float: A score how well it matches the gt or criterion
+    
+    Examples:
+        >>> judge("I have 5 apples", criterion="Has more than 1 apple")
+        5.0
+        
+        >>> judge("I have coal", gt="I have some coal")
+        5.0
+        
+        >>> judge("No apples here", criterion="Has more than 1 apple")
+        0.0
+        
+        >>> judge("I have 5 apples", criterion="Has apples", 
+        ...       prompt_template="Does '{text}' meet '{criterion}'? Score 0-1:")
+        5.0
+    """    
+    # Build the prompt based on what's provided
+    if prompt_template:
+        # Use custom prompt template
+        prompt = prompt_template.format(criterion=criterion, gt=gt)
+    elif criterion and gt:
+        raise ValueError("Criterion and gt cannot be set at the same time without a custom template.")
+    elif criterion:
+        # Load criterion prompt template
+        template = _load_prompt_template(_PROMPTS_DIR / "prompt_criterion.md")
+        prompt = template.format(criterion=criterion)
+    elif gt:
+        # Load ground truth prompt template
+        template = _load_prompt_template(_PROMPTS_DIR / "prompt_gt.md")
+        prompt = template.format(gt=gt)
+    else:
+        raise ValueError("Either criterion or gt must be provided")
+    
+    # Make the API call
+    client = _get_client()
+    response = client.chat.completions.create(
+        model=os.getenv("OPENAI_MODEL_NAME", "gpt-4o"),
+        messages=[
+            {"role": "system", "content": prompt},
+            {"role": "user", "content": text}
+        ],
+        temperature=0.0,
+    )
+    
+    # Extract and return the score
+    content = response.choices[0].message.content
+    if content is None:
+        raise ValueError("Received empty response from API")
+    
+    score_text = content.strip()
+    try:
+        score = float(score_text)
+        # Clamp the score between 0.0 and 1.0
+        return score
+    except ValueError:
+        raise ValueError(f"Failed to parse score from response: {score_text}")

quack_test-0.1.0/quack_test.egg-info/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: quack-test
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Dynamic: license-file
+
+# Quack Test
+
+A plugin for pytest to evaluate non-deterministic agent components.
+
+## Installation
+
+Simply pip install it
+
+```bash
+pip install quack-test
+```
+
+## Usage
+
+You can simply specify `@nondeterministic` for components, which are not deterministic and should be run multiple times.
+
+```python
+import random
+from quack_test import nondeterministic_fixture, nondeterministic_test, judge
+
+@nondeterministic_fixture(n=5)
+def sample_text():
+    return f"I have {random.randint(10)} apples."
+
+@nondeterministic_test(score=0.8)
+def test_apples(sample_text):
+    return judge(sample_text, criterion="Has more than 1 apple")
+
+@nondeterministic_test(score=0.8)
+def test_coal(sample_text):
+    return judge(sample_text, criterion="Has only coal")
+```

quack_test-0.1.0/quack_test.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+LICENSE
+README.md
+pyproject.toml
+quack_test/__init__.py
+quack_test/decorators.py
+quack_test/judge.py
+quack_test.egg-info/PKG-INFO
+quack_test.egg-info/SOURCES.txt
+quack_test.egg-info/dependency_links.txt
+quack_test.egg-info/requires.txt
+quack_test.egg-info/top_level.txt
+test/test_example.py

quack_test-0.1.0/quack_test.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

quack_test-0.1.0/quack_test.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pytest>=7.0.0
+openai>=1.0.0
+python-dotenv>=1.0.0

quack_test-0.1.0/quack_test.egg-info/top_level.txt

@@ -0,0 +1 @@
+quack_test

quack_test-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

quack_test-0.1.0/test/test_example.py

@@ -0,0 +1,18 @@
+import random
+from quack_test import nondeterministic_fixture, nondeterministic_test, judge
+
+@nondeterministic_fixture(n=5)
+def sample_text():
+    return f"I have {random.randint(0, 10)} apples."
+
+@nondeterministic_test(score=0.8)
+def test_apples(sample_text):
+    return judge(sample_text, criterion="Has more than 1 apple")
+
+@nondeterministic_test(score=0.8)
+def test_apples_gt(sample_text):
+    return judge(sample_text, gt="I have N (0-10) apples")
+
+@nondeterministic_test(score=0.8, should_fail=True)
+def test_coal(sample_text):
+    return judge(sample_text, criterion="Has only coal")