evalwise 0.1.0__py3-none-any.whl

evalwise/image/assertions.py

@@ -0,0 +1,382 @@
+"""
+Image assertions - deterministic checks for generated images.
+
+These use vision models (CLIP, object detection, etc.) which are
+deterministic given the model weights.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any
+
+from evalwise.core.context import get_context
+from evalwise.core.result import AssertionResult, Status
+
+
+def _load_image(image: str | Path | Any) -> Any:
+    """Load image from path or return PIL Image."""
+    try:
+        from PIL import Image
+    except ImportError:
+        raise ImportError("Pillow required: pip install pillow")
+    
+    if isinstance(image, (str, Path)):
+        return Image.open(image)
+    return image
+
+
+class ImageAssert:
+    """
+    Deterministic assertions for generated images.
+    
+    Example:
+        from evalwise.image import ImageAssert
+        
+        ImageAssert.clip_score(image, "a cat sitting on a couch", threshold=0.25)
+        ImageAssert.contains_object(image, "cat")
+        ImageAssert.nsfw_below(image, threshold=0.1)
+    """
+    
+    # ==================== CLIP-based ====================
+    
+    @staticmethod
+    def clip_score(image: str | Path, prompt: str, *, threshold: float = 0.25) -> bool:
+        """
+        Assert image matches text prompt (CLIP similarity).
+        
+        Args:
+            image: Path to image or PIL Image
+            prompt: Text description
+            threshold: Minimum similarity score (0-1, typically 0.2-0.35 is good)
+        """
+        start = time.perf_counter()
+        
+        try:
+            import torch
+            import open_clip
+        except ImportError:
+            raise ImportError("open-clip-torch required: pip install open-clip-torch")
+        
+        # Load model
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            'ViT-B-32', pretrained='laion2b_s34b_b79k'
+        )
+        tokenizer = open_clip.get_tokenizer('ViT-B-32')
+        
+        # Load and preprocess image
+        img = _load_image(image)
+        img_tensor = preprocess(img).unsqueeze(0)
+        
+        # Encode
+        with torch.no_grad():
+            image_features = model.encode_image(img_tensor)
+            text_features = model.encode_text(tokenizer([prompt]))
+            
+            # Normalize
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+            text_features /= text_features.norm(dim=-1, keepdim=True)
+            
+            # Cosine similarity
+            similarity = (image_features @ text_features.T).item()
+        
+        passed = similarity >= threshold
+        
+        result = AssertionResult(
+            name="clip_score",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"CLIP score {similarity:.3f} < threshold {threshold}" if not passed else None,
+            expected=f">= {threshold}",
+            actual=round(similarity, 3),
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Metadata ====================
+    
+    @staticmethod
+    def resolution_is(image: str | Path, *, width: int, height: int) -> bool:
+        """Assert image has exact resolution."""
+        start = time.perf_counter()
+        
+        img = _load_image(image)
+        actual_width, actual_height = img.size
+        
+        passed = actual_width == width and actual_height == height
+        
+        result = AssertionResult(
+            name="resolution_is",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Resolution {actual_width}x{actual_height} != {width}x{height}" if not passed else None,
+            expected=f"{width}x{height}",
+            actual=f"{actual_width}x{actual_height}",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def resolution_min(image: str | Path, *, width: int, height: int) -> bool:
+        """Assert image meets minimum resolution."""
+        start = time.perf_counter()
+        
+        img = _load_image(image)
+        actual_width, actual_height = img.size
+        
+        passed = actual_width >= width and actual_height >= height
+        
+        result = AssertionResult(
+            name="resolution_min",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Resolution {actual_width}x{actual_height} below minimum {width}x{height}" if not passed else None,
+            expected=f">= {width}x{height}",
+            actual=f"{actual_width}x{actual_height}",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def aspect_ratio(image: str | Path, *, ratio: float, tolerance: float = 0.05) -> bool:
+        """Assert image has expected aspect ratio (width/height)."""
+        start = time.perf_counter()
+        
+        img = _load_image(image)
+        width, height = img.size
+        actual_ratio = width / height
+        
+        passed = abs(actual_ratio - ratio) <= tolerance
+        
+        result = AssertionResult(
+            name="aspect_ratio",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Aspect ratio {actual_ratio:.2f} != {ratio} (±{tolerance})" if not passed else None,
+            expected=f"{ratio} (±{tolerance})",
+            actual=round(actual_ratio, 2),
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    @staticmethod
+    def format_is(image: str | Path, expected_format: str) -> bool:
+        """Assert image format (PNG, JPEG, WEBP, etc.)."""
+        start = time.perf_counter()
+        
+        img = _load_image(image)
+        actual_format = img.format or "unknown"
+        
+        passed = actual_format.upper() == expected_format.upper()
+        
+        result = AssertionResult(
+            name="format_is",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Format {actual_format} != {expected_format}" if not passed else None,
+            expected=expected_format.upper(),
+            actual=actual_format,
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Object Detection ====================
+    
+    @staticmethod
+    def contains_object(
+        image: str | Path, 
+        object_name: str, 
+        *, 
+        confidence: float = 0.5,
+        model: str = "yolov8n",
+    ) -> bool:
+        """
+        Assert image contains a specific object.
+        
+        Uses YOLO for detection - deterministic given model weights.
+        """
+        start = time.perf_counter()
+        
+        try:
+            from ultralytics import YOLO
+        except ImportError:
+            raise ImportError("ultralytics required: pip install ultralytics")
+        
+        # Load model
+        detector = YOLO(model)
+        
+        # Run detection
+        img = _load_image(image)
+        results = detector(img, verbose=False)
+        
+        # Check for object
+        found = False
+        found_conf = 0.0
+        
+        for r in results:
+            for box in r.boxes:
+                class_name = detector.names[int(box.cls)]
+                conf = float(box.conf)
+                
+                if object_name.lower() in class_name.lower() and conf >= confidence:
+                    found = True
+                    found_conf = max(found_conf, conf)
+        
+        result = AssertionResult(
+            name="contains_object",
+            status=Status.PASS if found else Status.FAIL,
+            message=f"Object '{object_name}' not found with confidence >= {confidence}" if not found else None,
+            expected=f"{object_name} (>= {confidence})",
+            actual=f"found with {found_conf:.2f}" if found else "not found",
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return found
+    
+    @staticmethod
+    def object_count(
+        image: str | Path,
+        object_name: str,
+        *,
+        min_count: int | None = None,
+        max_count: int | None = None,
+        exactly: int | None = None,
+        confidence: float = 0.5,
+    ) -> bool:
+        """Assert number of objects of a type in image."""
+        start = time.perf_counter()
+        
+        try:
+            from ultralytics import YOLO
+        except ImportError:
+            raise ImportError("ultralytics required: pip install ultralytics")
+        
+        detector = YOLO("yolov8n")
+        img = _load_image(image)
+        results = detector(img, verbose=False)
+        
+        count = 0
+        for r in results:
+            for box in r.boxes:
+                class_name = detector.names[int(box.cls)]
+                conf = float(box.conf)
+                if object_name.lower() in class_name.lower() and conf >= confidence:
+                    count += 1
+        
+        if exactly is not None:
+            passed = count == exactly
+            expected = f"exactly {exactly}"
+        elif min_count is not None and max_count is not None:
+            passed = min_count <= count <= max_count
+            expected = f"{min_count}-{max_count}"
+        elif min_count is not None:
+            passed = count >= min_count
+            expected = f">= {min_count}"
+        elif max_count is not None:
+            passed = count <= max_count
+            expected = f"<= {max_count}"
+        else:
+            passed = True
+            expected = "any"
+        
+        result = AssertionResult(
+            name="object_count",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Found {count} '{object_name}', expected {expected}" if not passed else None,
+            expected=expected,
+            actual=count,
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Safety ====================
+    
+    @staticmethod
+    def nsfw_below(image: str | Path, *, threshold: float = 0.5) -> bool:
+        """
+        Assert NSFW score is below threshold.
+        
+        Uses a safety classifier - deterministic given model.
+        """
+        start = time.perf_counter()
+        
+        try:
+            from transformers import pipeline
+        except ImportError:
+            raise ImportError("transformers required: pip install transformers")
+        
+        classifier = pipeline(
+            "image-classification",
+            model="Falconsai/nsfw_image_detection",
+        )
+        
+        img = _load_image(image)
+        results = classifier(img)
+        
+        nsfw_score = 0.0
+        for r in results:
+            if r["label"].lower() == "nsfw":
+                nsfw_score = r["score"]
+                break
+        
+        passed = nsfw_score < threshold
+        
+        result = AssertionResult(
+            name="nsfw_below",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"NSFW score {nsfw_score:.3f} >= threshold {threshold}" if not passed else None,
+            expected=f"< {threshold}",
+            actual=round(nsfw_score, 3),
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed
+    
+    # ==================== Similarity ====================
+    
+    @staticmethod
+    def image_similarity(
+        image: str | Path,
+        reference: str | Path,
+        *,
+        threshold: float = 0.8,
+    ) -> bool:
+        """Assert image is similar to reference image (CLIP embedding similarity)."""
+        start = time.perf_counter()
+        
+        try:
+            import torch
+            import open_clip
+        except ImportError:
+            raise ImportError("open-clip-torch required: pip install open-clip-torch")
+        
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            'ViT-B-32', pretrained='laion2b_s34b_b79k'
+        )
+        
+        img1 = preprocess(_load_image(image)).unsqueeze(0)
+        img2 = preprocess(_load_image(reference)).unsqueeze(0)
+        
+        with torch.no_grad():
+            feat1 = model.encode_image(img1)
+            feat2 = model.encode_image(img2)
+            
+            feat1 /= feat1.norm(dim=-1, keepdim=True)
+            feat2 /= feat2.norm(dim=-1, keepdim=True)
+            
+            similarity = (feat1 @ feat2.T).item()
+        
+        passed = similarity >= threshold
+        
+        result = AssertionResult(
+            name="image_similarity",
+            status=Status.PASS if passed else Status.FAIL,
+            message=f"Similarity {similarity:.3f} < threshold {threshold}" if not passed else None,
+            expected=f">= {threshold}",
+            actual=round(similarity, 3),
+            duration_ms=(time.perf_counter() - start) * 1000,
+        )
+        get_context().add_assertion(result)
+        return passed

evalwise/text/__init__.py

@@ -0,0 +1,5 @@
+"""Text assertion module - deterministic checks for text/LLM outputs."""
+
+from evalwise.text.assertions import Assert
+
+__all__ = ["Assert"]