evalwise 0.1.0__py3-none-any.whl

evalwise/__init__.py

@@ -0,0 +1,23 @@
+"""
+EvalWise - Deterministic-first AI evaluation for text, image, audio, and video.
+
+The eval SDK that doesn't default to "ask another AI if this is good."
+"""
+
+from evalwise.core.suite import Suite
+from evalwise.core.result import TestResult, SuiteResult
+from evalwise.core.dataset import Dataset
+from evalwise.text import Assert as TextAssert
+
+# Convenience alias
+Assert = TextAssert
+
+__version__ = "0.1.0"
+__all__ = [
+    "Suite",
+    "TestResult",
+    "SuiteResult", 
+    "Dataset",
+    "Assert",
+    "TextAssert",
+]

evalwise/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running as `python -m evalwise`."""
+
+from evalwise.cli import main
+
+if __name__ == "__main__":
+    main()

evalwise/audio/__init__.py

@@ -0,0 +1,5 @@
+"""Audio assertion module - deterministic checks for generated audio."""
+
+from evalwise.audio.assertions import AudioAssert
+
+__all__ = ["AudioAssert"]

evalwise/audio/assertions.py

@@ -0,0 +1,335 @@
+"""
+Audio assertions - deterministic checks for generated audio.
+
+Uses Whisper for transcription, librosa for analysis.
+"""
+
+from __future__ import annotations
+
+import time
+import urllib.error
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from evalwise.core.context import get_context
+from evalwise.core.result import AssertionResult, Status
+
+
+_FFMPEG_INSTALL_HELP = """\
+ffmpeg is required by Whisper but was not found on your system.
+
+Install it for your platform:
+
+  macOS:     brew install ffmpeg
+  Ubuntu:    sudo apt update && sudo apt install ffmpeg
+  Windows:   winget install Gyan.FFmpeg
+             or download from https://ffmpeg.org/download.html
+
+After installing, verify it is on your PATH:
+
+  ffmpeg -version
+"""
+
+_SSL_DOWNLOAD_HELP = """\
+Failed to download the Whisper model because of an SSL certificate verification error.
+
+This is usually a local environment issue, not a problem with EvalWise or Whisper.
+
+Fix options:
+
+  1. Update certificates:
+       pip install --upgrade certifi
+       export SSL_CERT_FILE=$(python -m certifi)
+
+  2. Pre-download the model so no network call is needed:
+       Download the .pt file from https://github.com/openai/whisper/blob/main/model-card.md
+       and place it in ~/.cache/whisper/
+
+  3. (Local dev only) Disable verification:
+       export PYTHONHTTPSVERIFY=0
+"""
+
+
+@lru_cache(maxsize=None)
+def _load_whisper_model(model_name: str):
+    """Load a Whisper model, caching it to avoid repeated downloads."""
+    try:
+        import whisper
+    except ImportError:
+        raise ImportError("openai-whisper required: pip install openai-whisper")
+
+    try:
+        return whisper.load_model(model_name)
+    except urllib.error.URLError as exc:
+        error_msg = str(exc).lower()
+        if "certificate" in error_msg or "ssl" in error_msg:
+            raise RuntimeError(_SSL_DOWNLOAD_HELP) from exc
+        raise
+
+
+def _handle_whisper_call(exc: Exception) -> None:
+    """Convert common Whisper runtime errors into actionable messages."""
+    if isinstance(exc, FileNotFoundError) and "ffmpeg" in str(exc):
+        raise RuntimeError(_FFMPEG_INSTALL_HELP) from exc
+    raise exc
+
+
+class AudioAssert:
+    """
+    Deterministic assertions for generated audio.
+    
+    Example:
+        from evalwise.audio import AudioAssert
+        
+        AudioAssert.transcription_contains(audio, "hello world")
+        AudioAssert.duration_between(audio, min_sec=5, max_sec=30)
+        AudioAssert.language_is(audio, "en")
+    """
+    
+    # ==================== Transcription ====================
+    
+    @staticmethod
+    def transcription_equals(
+        audio: str | Path,
+        expected: str,
+        *,
+        model: str = "base",
+        case_sensitive: bool = False,
+    ) -> bool:
+        """Assert audio transcription equals expected text."""
+        start = time.perf_counter()
+        
+        model_obj = _load_whisper_model(model)
+        try:
+            result_whisper = model_obj.transcribe(str(audio))
+        except Exception as exc:
+            _handle_whisper_call(exc)
+        transcript = result_whisper["text"].strip()
+        
+        if case_sensitive:
+            passed = transcript == expected
+        else:
+            passed = transcript.lower() == expected.lower()
+        
+        result = AssertionResult(
+            name="transcription_equals",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Transcription mismatch" if not passed else None,
+            expected=expected,
+            actual=transcript,
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def transcription_contains(
+        audio: str | Path,
+        substring: str,
+        *,
+        model: str = "base",
+        case_sensitive: bool = False,
+    ) -> bool:
+        """Assert audio transcription contains substring."""
+        start = time.perf_counter()
+        
+        model_obj = _load_whisper_model(model)
+        try:
+            result_whisper = model_obj.transcribe(str(audio))
+        except Exception as exc:
+            _handle_whisper_call(exc)
+        transcript = result_whisper["text"].strip()
+        
+        if case_sensitive:
+            passed = substring in transcript
+        else:
+            passed = substring.lower() in transcript.lower()
+        
+        result = AssertionResult(
+            name="transcription_contains",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"'{substring}' not in transcription" if not passed else None,
+            expected=f"contains '{substring}'",
+            actual=transcript[:200] + "..." if len(transcript) > 200 else transcript,
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def language_is(audio: str | Path, expected_lang: str, *, model: str = "base") -> bool:
+        """Assert detected language matches expected."""
+        start = time.perf_counter()
+        
+        try:
+            import whisper
+        except ImportError:
+            raise ImportError("openai-whisper required: pip install openai-whisper")
+        
+        model_obj = _load_whisper_model(model)
+        
+        # Load audio and detect language
+        try:
+            audio_data = whisper.load_audio(str(audio))
+        except Exception as exc:
+            _handle_whisper_call(exc)
+        audio_data = whisper.pad_or_trim(audio_data)
+        mel = whisper.log_mel_spectrogram(audio_data).to(model_obj.device)
+        
+        _, probs = model_obj.detect_language(mel)
+        detected = max(probs, key=probs.get)
+        
+        passed = detected == expected_lang
+        
+        result = AssertionResult(
+            name="language_is",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Detected '{detected}', expected '{expected_lang}'" if not passed else None,
+            expected=expected_lang,
+            actual=detected,
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Duration ====================
+    
+    @staticmethod
+    def _get_duration(audio: str | Path) -> float:
+        """Get audio duration using available library."""
+        audio_path = str(audio)
+        
+        # Try scipy first (more commonly available)
+        try:
+            from scipy.io import wavfile
+            if audio_path.endswith('.wav'):
+                sr, data = wavfile.read(audio_path)
+                return len(data) / sr
+        except ImportError:
+            pass
+        except Exception:
+            pass  # Not a WAV or failed, try librosa
+        
+        # Fall back to librosa
+        try:
+            import librosa
+            return librosa.get_duration(path=audio_path)
+        except ImportError:
+            raise ImportError("librosa or scipy required: pip install librosa (or scipy for WAV)")
+    
+    @staticmethod
+    def duration_between(
+        audio: str | Path,
+        *,
+        min_sec: float | None = None,
+        max_sec: float | None = None,
+    ) -> bool:
+        """Assert audio duration is within bounds."""
+        start = time.perf_counter()
+        
+        duration = AudioAssert._get_duration(audio)
+        
+        if min_sec is not None and max_sec is not None:
+            passed = min_sec <= duration <= max_sec
+            expected = f"{min_sec}-{max_sec}s"
+        elif min_sec is not None:
+            passed = duration >= min_sec
+            expected = f">= {min_sec}s"
+        elif max_sec is not None:
+            passed = duration <= max_sec
+            expected = f"<= {max_sec}s"
+        else:
+            passed = True
+            expected = "any"
+        
+        result = AssertionResult(
+            name="duration_between",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Duration {duration:.1f}s not in range {expected}" if not passed else None,
+            expected=expected,
+            actual=f"{duration:.1f}s",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def duration_is(audio: str | Path, *, seconds: float, tolerance: float = 0.5) -> bool:
+        """Assert audio duration matches expected."""
+        start = time.perf_counter()
+        
+        duration = AudioAssert._get_duration(audio)
+        passed = abs(duration - seconds) <= tolerance
+        
+        result = AssertionResult(
+            name="duration_is",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Duration {duration:.1f}s != {seconds}s (±{tolerance})" if not passed else None,
+            expected=f"{seconds}s (±{tolerance})",
+            actual=f"{duration:.1f}s",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Quality ====================
+    
+    @staticmethod
+    def sample_rate_is(audio: str | Path, *, hz: int) -> bool:
+        """Assert audio sample rate."""
+        start = time.perf_counter()
+        
+        try:
+            import librosa
+        except ImportError:
+            raise ImportError("librosa required: pip install librosa")
+        
+        _, sr = librosa.load(str(audio), sr=None)
+        passed = sr == hz
+        
+        result = AssertionResult(
+            name="sample_rate_is",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Sample rate {sr}Hz != {hz}Hz" if not passed else None,
+            expected=f"{hz}Hz",
+            actual=f"{sr}Hz",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def no_silence(audio: str | Path, *, max_silence_sec: float = 1.0) -> bool:
+        """Assert no long silence gaps in audio."""
+        start = time.perf_counter()
+        
+        try:
+            import librosa
+            import numpy as np
+        except ImportError:
+            raise ImportError("librosa required: pip install librosa")
+        
+        y, sr = librosa.load(str(audio))
+        
+        # Find non-silent intervals
+        intervals = librosa.effects.split(y, top_db=30)
+        
+        # Check gaps between intervals
+        max_gap = 0.0
+        for i in range(1, len(intervals)):
+            gap = (intervals[i][0] - intervals[i-1][1]) / sr
+            max_gap = max(max_gap, gap)
+        
+        passed = max_gap <= max_silence_sec
+        
+        result = AssertionResult(
+            name="no_silence",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Max silence gap {max_gap:.1f}s > {max_silence_sec}s" if not passed else None,
+            expected=f"<= {max_silence_sec}s silence",
+            actual=f"{max_gap:.1f}s max gap",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed

evalwise/cli.py

@@ -0,0 +1,251 @@
+"""Command-line interface for evalwise."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+@click.group()
+@click.version_option()
+def main() -> None:
+    """EvalWise - Deterministic-first AI evaluation."""
+    pass
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--dataset", "-d", type=click.Path(exists=True), help="Dataset file (JSON/YAML)")
+@click.option("--threshold", "-t", type=float, default=0.95, help="Pass rate threshold")
+@click.option("--output", "-o", type=click.Path(), help="Output file for results (JSON)")
+@click.option("--fail-fast", is_flag=True, help="Stop on first failure")
+@click.option("--ci", is_flag=True, help="CI mode: exit code 1 on failure")
+@click.option("--tags", help="Only run tests with these tags (comma-separated)")
+@click.option("--exclude-tags", help="Skip tests with these tags (comma-separated)")
+def run(
+    path: str,
+    dataset: str | None,
+    threshold: float,
+    output: str | None,
+    fail_fast: bool,
+    ci: bool,
+    tags: str | None,
+    exclude_tags: str | None,
+) -> None:
+    """
+    Run evaluation suite.
+    
+    PATH can be a Python file containing a Suite, or a directory of test files.
+    """
+    from evalwise.core.suite import Suite
+    
+    path_obj = Path(path)
+    
+    # Find and load suites
+    suites: list[Suite] = []
+    
+    if path_obj.is_file():
+        suites.extend(_load_suites_from_file(path_obj))
+    elif path_obj.is_dir():
+        for f in path_obj.glob("**/*.py"):
+            if f.name.startswith("test_") or f.name.endswith("_test.py"):
+                suites.extend(_load_suites_from_file(f))
+    
+    if not suites:
+        console.print("[red]No test suites found[/red]")
+        sys.exit(1)
+    
+    # Load dataset if provided
+    dataset_items = None
+    if dataset:
+        dataset_items = _load_dataset(dataset)
+    
+    # Parse tags
+    tags_list = [t.strip() for t in tags.split(",")] if tags else None
+    exclude_tags_list = [t.strip() for t in exclude_tags.split(",")] if exclude_tags else None
+    
+    # Run all suites
+    all_passed = True
+    all_results = []
+    
+    for suite in suites:
+        console.print(f"\n[bold]Running suite: {suite.name}[/bold]")
+        
+        result = suite.run(
+            dataset=dataset_items, 
+            fail_fast=fail_fast,
+            tags=tags_list,
+            exclude_tags=exclude_tags_list,
+        )
+        all_results.append(result)
+        
+        # Print results
+        _print_results(result)
+        
+        if result.pass_rate < threshold:
+            all_passed = False
+            console.print(
+                f"[red]✗ Pass rate {result.pass_rate:.1%} below threshold {threshold:.1%}[/red]"
+            )
+        else:
+            console.print(
+                f"[green]✓ Pass rate {result.pass_rate:.1%} meets threshold {threshold:.1%}[/green]"
+            )
+    
+    # Save output if requested
+    if output:
+        output_data = [r.to_dict() for r in all_results]
+        with open(output, "w") as f:
+            json.dump(output_data, f, indent=2)
+        console.print(f"\nResults saved to {output}")
+    
+    # Exit code for CI
+    if ci and not all_passed:
+        sys.exit(1)
+
+
+def _load_suites_from_file(path: Path) -> list:
+    """Load Suite objects from a Python file."""
+    from evalwise.core.suite import Suite
+    
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    if spec is None or spec.loader is None:
+        return []
+    
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[path.stem] = module
+    spec.loader.exec_module(module)
+    
+    suites = []
+    for name in dir(module):
+        obj = getattr(module, name)
+        if isinstance(obj, Suite):
+            suites.append(obj)
+    
+    return suites
+
+
+def _load_dataset(path: str) -> list:
+    """Load dataset from file."""
+    p = Path(path)
+    
+    if p.suffix == ".json":
+        with open(p) as f:
+            data = json.load(f)
+    elif p.suffix in (".yaml", ".yml"):
+        import yaml
+        with open(p) as f:
+            data = yaml.safe_load(f)
+    elif p.suffix == ".jsonl":
+        data = []
+        with open(p) as f:
+            for line in f:
+                if line.strip():
+                    data.append(json.loads(line))
+    else:
+        raise ValueError(f"Unsupported format: {p.suffix}")
+    
+    if isinstance(data, list):
+        return data
+    elif isinstance(data, dict) and "tests" in data:
+        return data["tests"]
+    else:
+        raise ValueError("Dataset must be a list or object with 'tests' key")
+
+
+def _print_results(result) -> None:
+    """Print suite results as a table."""
+    from evalwise.core.result import Status
+    
+    # Group by test name
+    test_groups: dict[str, list] = {}
+    for t in result.tests:
+        if t.test_name not in test_groups:
+            test_groups[t.test_name] = []
+        test_groups[t.test_name].append(t)
+    
+    for test_name, tests in test_groups.items():
+        passed = sum(1 for t in tests if t.passed)
+        total = len(tests)
+        
+        # Check for errors (exceptions during test)
+        errors = [t for t in tests if t.error]
+        if errors:
+            console.print(f"\n[red]✗ {test_name}: {len(errors)}/{total} had errors[/red]")
+            # Show first error as example
+            console.print(f"[dim]  Example error: {errors[0].error.split(chr(10))[0]}[/dim]")
+            continue
+        
+        # Collect assertion stats
+        assertion_stats: dict[str, tuple[int, int]] = {}
+        for t in tests:
+            for a in t.assertions:
+                if a.name not in assertion_stats:
+                    assertion_stats[a.name] = (0, 0)
+                p, tot = assertion_stats[a.name]
+                assertion_stats[a.name] = (p + (1 if a.passed else 0), tot + 1)
+        
+        if not assertion_stats:
+            console.print(f"\n[yellow]⚠ {test_name}: No assertions recorded[/yellow]")
+            continue
+        
+        table = Table(title=f"{test_name} ({passed}/{total} passed)")
+        table.add_column("Assertion")
+        table.add_column("Passed", justify="right")
+        table.add_column("Total", justify="right")
+        table.add_column("Rate", justify="right")
+        
+        for name, (p, tot) in assertion_stats.items():
+            rate = p / tot if tot > 0 else 0
+            color = "green" if rate >= 0.95 else "yellow" if rate >= 0.8 else "red"
+            table.add_row(name, str(p), str(tot), f"[{color}]{rate:.1%}[/{color}]")
+        
+        console.print(table)
+
+
+@main.command()
+def init() -> None:
+    """Create a sample eval file."""
+    sample = '''"""Sample evaluation suite."""
+
+from evalwise import Suite, Assert
+
+suite = Suite("my_evals")
+
+
+@suite.test
+def test_format(response: str):
+    """Test response format."""
+    Assert.word_count(response, min=10, max=500)
+    Assert.json_valid(response)
+
+
+@suite.test
+def test_content(response: str, expected: str):
+    """Test response content."""
+    Assert.contains(response, expected)
+
+
+# Run with: evalwise run test_eval.py --dataset golden.json
+'''
+    
+    with open("test_eval.py", "w") as f:
+        f.write(sample)
+    
+    console.print("[green]Created test_eval.py[/green]")
+    console.print("\nNext steps:")
+    console.print("  1. Edit test_eval.py to add your assertions")
+    console.print("  2. Create a dataset file (golden.json)")
+    console.print("  3. Run: evalwise run test_eval.py --dataset golden.json")
+
+
+if __name__ == "__main__":
+    main()

evalwise/core/__init__.py

@@ -0,0 +1,7 @@
+"""Core evaluation framework."""
+
+from evalwise.core.suite import Suite
+from evalwise.core.result import TestResult, SuiteResult, AssertionResult
+from evalwise.core.dataset import Dataset
+
+__all__ = ["Suite", "TestResult", "SuiteResult", "AssertionResult", "Dataset"]

evalwise/core/context.py

@@ -0,0 +1,47 @@
+"""Evaluation context for collecting assertion results."""
+
+from __future__ import annotations
+
+import contextvars
+from dataclasses import dataclass, field
+
+from evalwise.core.result import AssertionResult
+
+# Context variable to hold the current eval context
+_current_context: contextvars.ContextVar[EvalContext | None] = contextvars.ContextVar(
+    "eval_context", default=None
+)
+
+
+@dataclass
+class EvalContext:
+    """
+    Context object that collects assertion results during a test run.
+    
+    Assertions register their results here so the test runner can collect them.
+    """
+    
+    assertions: list[AssertionResult] = field(default_factory=list)
+    
+    def add_assertion(self, result: AssertionResult) -> None:
+        """Add an assertion result to this context."""
+        self.assertions.append(result)
+
+
+def get_context() -> EvalContext:
+    """Get the current evaluation context, creating one if needed."""
+    ctx = _current_context.get()
+    if ctx is None:
+        ctx = EvalContext()
+        _current_context.set(ctx)
+    return ctx
+
+
+def set_context(ctx: EvalContext) -> None:
+    """Set the current evaluation context."""
+    _current_context.set(ctx)
+
+
+def clear_context() -> None:
+    """Clear the current evaluation context."""
+    _current_context.set(None)