evalwise 0.1.0__py3-none-any.whl

evalwise/core/dataset.py

@@ -0,0 +1,102 @@
+"""Dataset loading and management."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterator
+
+
+class Dataset:
+    """
+    A collection of test inputs for evaluation.
+    
+    Example:
+        # From list
+        dataset = Dataset([
+            {"response": "Hello", "expected": "greeting"},
+            {"response": "Bye", "expected": "farewell"},
+        ])
+        
+        # From file
+        dataset = Dataset.from_file("./golden_set.json")
+        
+        # From CSV
+        dataset = Dataset.from_csv("./test_cases.csv")
+    """
+    
+    def __init__(self, items: list[dict[str, Any]]):
+        self.items = items
+    
+    def __len__(self) -> int:
+        return len(self.items)
+    
+    def __iter__(self) -> Iterator[dict[str, Any]]:
+        return iter(self.items)
+    
+    def __getitem__(self, idx: int) -> dict[str, Any]:
+        return self.items[idx]
+    
+    @classmethod
+    def from_file(cls, path: str | Path) -> Dataset:
+        """Load dataset from JSON or YAML file."""
+        p = Path(path)
+        
+        if p.suffix == ".json":
+            with open(p) as f:
+                data = json.load(f)
+        elif p.suffix in (".yaml", ".yml"):
+            try:
+                import yaml
+                with open(p) as f:
+                    data = yaml.safe_load(f)
+            except ImportError:
+                raise ImportError("PyYAML required: pip install pyyaml")
+        else:
+            raise ValueError(f"Unsupported format: {p.suffix}")
+        
+        if isinstance(data, list):
+            return cls(data)
+        elif isinstance(data, dict) and "tests" in data:
+            return cls(data["tests"])
+        else:
+            raise ValueError("File must contain a list or object with 'tests' key")
+    
+    @classmethod
+    def from_csv(cls, path: str | Path, **kwargs: Any) -> Dataset:
+        """Load dataset from CSV file."""
+        import csv
+        
+        items = []
+        with open(path, newline="") as f:
+            reader = csv.DictReader(f, **kwargs)
+            for row in reader:
+                items.append(dict(row))
+        
+        return cls(items)
+    
+    @classmethod
+    def from_jsonl(cls, path: str | Path) -> Dataset:
+        """Load dataset from JSON Lines file."""
+        items = []
+        with open(path) as f:
+            for line in f:
+                if line.strip():
+                    items.append(json.loads(line))
+        return cls(items)
+    
+    def filter(self, fn: callable) -> Dataset:
+        """Filter items by predicate function."""
+        return Dataset([item for item in self.items if fn(item)])
+    
+    def sample(self, n: int, seed: int | None = None) -> Dataset:
+        """Random sample of n items."""
+        import random
+        if seed is not None:
+            random.seed(seed)
+        sampled = random.sample(self.items, min(n, len(self.items)))
+        return Dataset(sampled)
+    
+    def to_list(self) -> list[dict[str, Any]]:
+        """Convert to list of dicts."""
+        return self.items.copy()

evalwise/core/result.py

@@ -0,0 +1,142 @@
+"""Result types for evaluations."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+
+class Status(Enum):
+    PASS = "pass"
+    FAIL = "fail"
+    ERROR = "error"
+    SKIP = "skip"
+
+
+@dataclass
+class AssertionResult:
+    """Result of a single assertion."""
+    
+    name: str
+    status: Status
+    message: str | None = None
+    expected: Any = None
+    actual: Any = None
+    duration_ms: float = 0.0
+    
+    @property
+    def passed(self) -> bool:
+        return self.status == Status.PASS
+    
+    def __repr__(self) -> str:
+        icon = "✓" if self.passed else "✗"
+        return f"{icon} {self.name}: {self.status.value}"
+
+
+@dataclass
+class TestResult:
+    """Result of a single test case."""
+    
+    test_name: str
+    input_id: str | None = None
+    assertions: list[AssertionResult] = field(default_factory=list)
+    duration_ms: float = 0.0
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    
+    @property
+    def status(self) -> Status:
+        if self.error:
+            return Status.ERROR
+        if not self.assertions:
+            return Status.SKIP
+        if all(a.passed for a in self.assertions):
+            return Status.PASS
+        return Status.FAIL
+    
+    @property
+    def passed(self) -> bool:
+        return self.status == Status.PASS
+    
+    @property
+    def pass_count(self) -> int:
+        return sum(1 for a in self.assertions if a.passed)
+    
+    @property
+    def fail_count(self) -> int:
+        return sum(1 for a in self.assertions if not a.passed)
+
+
+@dataclass
+class SuiteResult:
+    """Result of running a full test suite."""
+    
+    suite_name: str
+    tests: list[TestResult] = field(default_factory=list)
+    started_at: datetime = field(default_factory=datetime.now)
+    completed_at: datetime | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    
+    @property
+    def total_tests(self) -> int:
+        return len(self.tests)
+    
+    @property
+    def passed_tests(self) -> int:
+        return sum(1 for t in self.tests if t.passed)
+    
+    @property
+    def failed_tests(self) -> int:
+        return sum(1 for t in self.tests if t.status == Status.FAIL)
+    
+    @property
+    def error_tests(self) -> int:
+        return sum(1 for t in self.tests if t.status == Status.ERROR)
+    
+    @property
+    def pass_rate(self) -> float:
+        if not self.tests:
+            return 0.0
+        return self.passed_tests / self.total_tests
+    
+    @property
+    def duration_ms(self) -> float:
+        return sum(t.duration_ms for t in self.tests)
+    
+    def assert_pass_rate(self, threshold: float = 0.95) -> None:
+        """Raise AssertionError if pass rate is below threshold."""
+        if self.pass_rate < threshold:
+            raise AssertionError(
+                f"Pass rate {self.pass_rate:.1%} is below threshold {threshold:.1%}. "
+                f"({self.passed_tests}/{self.total_tests} tests passed)"
+            )
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "suite_name": self.suite_name,
+            "pass_rate": self.pass_rate,
+            "total_tests": self.total_tests,
+            "passed": self.passed_tests,
+            "failed": self.failed_tests,
+            "errors": self.error_tests,
+            "duration_ms": self.duration_ms,
+            "started_at": self.started_at.isoformat(),
+            "completed_at": self.completed_at.isoformat() if self.completed_at else None,
+            "tests": [
+                {
+                    "name": t.test_name,
+                    "input_id": t.input_id,
+                    "status": t.status.value,
+                    "assertions": [
+                        {
+                            "name": a.name,
+                            "status": a.status.value,
+                            "message": a.message,
+                        }
+                        for a in t.assertions
+                    ],
+                }
+                for t in self.tests
+            ],
+        }

evalwise/core/suite.py

@@ -0,0 +1,243 @@
+"""Test suite definition and runner."""
+
+from __future__ import annotations
+
+import time
+import traceback
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable
+
+from evalwise.core.result import AssertionResult, Status, SuiteResult, TestResult
+from evalwise.core.context import EvalContext, get_context, set_context
+
+
+@dataclass
+class TestCase:
+    """A single test case definition."""
+    
+    name: str
+    fn: Callable[..., None]
+    tags: list[str] = field(default_factory=list)
+
+
+class Suite:
+    """
+    A collection of evaluation tests.
+    
+    Example:
+        suite = Suite("summarization")
+        
+        @suite.test
+        def test_format(response, context):
+            Assert.bullet_count(response, exactly=3)
+            Assert.word_count(response, max=200)
+        
+        results = suite.run(dataset="./golden.json")
+    """
+    
+    def __init__(self, name: str):
+        self.name = name
+        self.tests: list[TestCase] = []
+        self._setup_fn: Callable[[], None] | None = None
+        self._teardown_fn: Callable[[], None] | None = None
+    
+    def test(
+        self, 
+        fn: Callable[..., None] | None = None,
+        *,
+        tags: list[str] | None = None,
+    ) -> Callable[..., None]:
+        """
+        Decorator to register a test function.
+        
+        Example:
+            @suite.test
+            def test_something(response):
+                Assert.contains(response, "hello")
+            
+            @suite.test(tags=["slow"])
+            def test_expensive(response):
+                Assert.llm_judge(response, "Is helpful")
+        """
+        def decorator(f: Callable[..., None]) -> Callable[..., None]:
+            self.tests.append(TestCase(
+                name=f.__name__,
+                fn=f,
+                tags=tags or [],
+            ))
+            return f
+        
+        if fn is not None:
+            return decorator(fn)
+        return decorator
+    
+    def setup(self, fn: Callable[[], None]) -> Callable[[], None]:
+        """Register a setup function to run before all tests."""
+        self._setup_fn = fn
+        return fn
+    
+    def teardown(self, fn: Callable[[], None]) -> Callable[[], None]:
+        """Register a teardown function to run after all tests."""
+        self._teardown_fn = fn
+        return fn
+    
+    def run(
+        self,
+        dataset: list[dict[str, Any]] | str | None = None,
+        *,
+        tags: list[str] | None = None,
+        exclude_tags: list[str] | None = None,
+        fail_fast: bool = False,
+    ) -> SuiteResult:
+        """
+        Run all tests in the suite.
+        
+        Args:
+            dataset: List of test inputs, or path to JSON/YAML file
+            tags: Only run tests with these tags
+            exclude_tags: Skip tests with these tags
+            fail_fast: Stop on first failure
+        
+        Returns:
+            SuiteResult with all test outcomes
+        """
+        result = SuiteResult(
+            suite_name=self.name,
+            started_at=datetime.now(),
+        )
+        
+        # Load dataset if path provided
+        if isinstance(dataset, str):
+            dataset = self._load_dataset(dataset)
+        
+        # Default to single empty input if no dataset
+        if not dataset:
+            dataset = [{}]
+        
+        # Filter tests by tags
+        tests_to_run = self._filter_tests(tags, exclude_tags)
+        
+        # Run setup
+        if self._setup_fn:
+            self._setup_fn()
+        
+        try:
+            for test_case in tests_to_run:
+                for i, input_data in enumerate(dataset):
+                    test_result = self._run_single_test(
+                        test_case, 
+                        input_data,
+                        input_id=str(i),
+                    )
+                    result.tests.append(test_result)
+                    
+                    if fail_fast and not test_result.passed:
+                        break
+                
+                if fail_fast and result.failed_tests > 0:
+                    break
+        finally:
+            if self._teardown_fn:
+                self._teardown_fn()
+        
+        result.completed_at = datetime.now()
+        return result
+    
+    def _run_single_test(
+        self, 
+        test_case: TestCase, 
+        input_data: dict[str, Any],
+        input_id: str,
+    ) -> TestResult:
+        """Run a single test case with given input."""
+        import inspect
+        
+        context = EvalContext()
+        set_context(context)
+        
+        start_time = time.perf_counter()
+        error: str | None = None
+        
+        try:
+            # Get function signature to filter kwargs
+            sig = inspect.signature(test_case.fn)
+            params = sig.parameters
+            
+            # Check if function accepts **kwargs
+            has_var_keyword = any(
+                p.kind == inspect.Parameter.VAR_KEYWORD 
+                for p in params.values()
+            )
+            
+            if has_var_keyword:
+                # Function accepts any kwargs
+                filtered_data = input_data
+            else:
+                # Only pass kwargs that match function parameters
+                filtered_data = {
+                    k: v for k, v in input_data.items() 
+                    if k in params
+                }
+            
+            # Call test function with filtered kwargs
+            test_case.fn(**filtered_data)
+        except AssertionError as e:
+            # Assertion failures are expected
+            pass
+        except Exception as e:
+            error = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
+        
+        duration_ms = (time.perf_counter() - start_time) * 1000
+        
+        return TestResult(
+            test_name=test_case.name,
+            input_id=input_id,
+            assertions=context.assertions,
+            duration_ms=duration_ms,
+            error=error,
+            metadata=input_data,
+        )
+    
+    def _filter_tests(
+        self, 
+        tags: list[str] | None,
+        exclude_tags: list[str] | None,
+    ) -> list[TestCase]:
+        """Filter tests by tags."""
+        result = self.tests
+        
+        if tags:
+            result = [t for t in result if any(tag in t.tags for tag in tags)]
+        
+        if exclude_tags:
+            result = [t for t in result if not any(tag in t.tags for tag in exclude_tags)]
+        
+        return result
+    
+    def _load_dataset(self, path: str) -> list[dict[str, Any]]:
+        """Load dataset from file."""
+        import json
+        from pathlib import Path
+        
+        p = Path(path)
+        
+        if p.suffix == ".json":
+            with open(p) as f:
+                data = json.load(f)
+        elif p.suffix in (".yaml", ".yml"):
+            try:
+                import yaml
+                with open(p) as f:
+                    data = yaml.safe_load(f)
+            except ImportError:
+                raise ImportError("PyYAML required for YAML datasets: pip install pyyaml")
+        else:
+            raise ValueError(f"Unsupported dataset format: {p.suffix}")
+        
+        if isinstance(data, list):
+            return data
+        elif isinstance(data, dict) and "tests" in data:
+            return data["tests"]
+        else:
+            raise ValueError("Dataset must be a list or object with 'tests' key")

evalwise/image/__init__.py

@@ -0,0 +1,5 @@
+"""Image assertion module - deterministic checks for generated images."""
+
+from evalwise.image.assertions import ImageAssert
+
+__all__ = ["ImageAssert"]