prela 0.1.0__py3-none-any.whl

prela/evals/assertions/semantic.py

@@ -0,0 +1,223 @@
+"""
+Semantic assertions using embedding-based similarity.
+
+Requires sentence-transformers library:
+    pip install sentence-transformers
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+# Check tier before allowing semantic assertions
+from prela.license import check_tier
+
+if not check_tier("Semantic assertions", "lunch-money", silent=False):
+    raise ImportError(
+        "Semantic assertions require 'lunch-money' subscription or higher. "
+        "Upgrade at https://prela.dev/pricing"
+    )
+
+try:
+    from sentence_transformers import SentenceTransformer
+    import numpy as np
+
+    SENTENCE_TRANSFORMERS_AVAILABLE = True
+except ImportError:
+    SENTENCE_TRANSFORMERS_AVAILABLE = False
+    SentenceTransformer = None
+    np = None
+
+from prela.core.span import Span
+from prela.evals.assertions.base import AssertionResult, BaseAssertion
+
+
+class SemanticSimilarityAssertion(BaseAssertion):
+    """Assert that output is semantically similar to expected text.
+
+    Uses sentence embeddings to compare semantic meaning rather than exact
+    text matching. Useful for evaluating LLM outputs where phrasing varies
+    but meaning should be consistent.
+
+    Example:
+        >>> assertion = SemanticSimilarityAssertion(
+        ...     expected_text="The weather is nice today",
+        ...     threshold=0.8
+        ... )
+        >>> result = assertion.evaluate(
+        ...     output="Today has beautiful weather",
+        ...     expected=None,
+        ...     trace=None
+        ... )
+        >>> assert result.passed  # High similarity despite different wording
+
+    Requires:
+        pip install sentence-transformers
+    """
+
+    # Class-level model cache (shared across instances)
+    _model_cache: dict[str, Any] = {}
+
+    # Embedding cache (to avoid recomputing for same text)
+    _embedding_cache: dict[str, Any] = {}
+
+    def __init__(
+        self,
+        expected_text: str,
+        threshold: float = 0.8,
+        model_name: str = "all-MiniLM-L6-v2",
+    ):
+        """Initialize semantic similarity assertion.
+
+        Args:
+            expected_text: Text to compare against
+            threshold: Minimum cosine similarity score (0-1) to pass
+            model_name: Sentence transformer model to use
+                       (default: all-MiniLM-L6-v2, fast and accurate)
+
+        Raises:
+            ImportError: If sentence-transformers is not installed
+            ValueError: If threshold is not between 0 and 1
+        """
+        if not SENTENCE_TRANSFORMERS_AVAILABLE:
+            raise ImportError(
+                "sentence-transformers required for semantic similarity. "
+                "Install with: pip install sentence-transformers"
+            )
+
+        if not 0 <= threshold <= 1:
+            raise ValueError(f"threshold must be between 0 and 1, got {threshold}")
+
+        self.expected_text = expected_text
+        self.threshold = threshold
+        self.model_name = model_name
+
+        # Load model (cached at class level)
+        if model_name not in self._model_cache:
+            self._model_cache[model_name] = SentenceTransformer(model_name)
+
+        self.model = self._model_cache[model_name]
+
+    def _get_embedding(self, text: str) -> Any:
+        """Get embedding for text, using cache if available."""
+        # Create cache key from text hash
+        text_hash = hashlib.md5(text.encode()).hexdigest()
+        cache_key = f"{self.model_name}:{text_hash}"
+
+        if cache_key not in self._embedding_cache:
+            self._embedding_cache[cache_key] = self.model.encode(
+                text,
+                convert_to_numpy=True,
+                normalize_embeddings=True,
+            )
+
+        return self._embedding_cache[cache_key]
+
+    def _cosine_similarity(self, embedding1: Any, embedding2: Any) -> float:
+        """Compute cosine similarity between two embeddings."""
+        # Embeddings are already normalized, so dot product = cosine similarity
+        return float(np.dot(embedding1, embedding2))
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output is semantically similar to expected text."""
+        output_str = str(output)
+
+        # Get embeddings
+        try:
+            output_embedding = self._get_embedding(output_str)
+            expected_embedding = self._get_embedding(self.expected_text)
+        except Exception as e:
+            return AssertionResult(
+                passed=False,
+                assertion_type="semantic_similarity",
+                message=f"Failed to compute embeddings: {e}",
+                expected=self.expected_text,
+                actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+                details={"error": str(e)},
+            )
+
+        # Compute similarity
+        similarity = self._cosine_similarity(output_embedding, expected_embedding)
+        passed = similarity >= self.threshold
+
+        # Determine score interpretation
+        if similarity >= 0.9:
+            interpretation = "very high"
+        elif similarity >= 0.8:
+            interpretation = "high"
+        elif similarity >= 0.7:
+            interpretation = "moderate"
+        elif similarity >= 0.6:
+            interpretation = "low"
+        else:
+            interpretation = "very low"
+
+        if passed:
+            message = (
+                f"Output is semantically similar to expected "
+                f"(similarity: {similarity:.3f}, {interpretation})"
+            )
+        else:
+            message = (
+                f"Output is not semantically similar enough "
+                f"(similarity: {similarity:.3f} < threshold: {self.threshold}, {interpretation})"
+            )
+
+        return AssertionResult(
+            passed=passed,
+            assertion_type="semantic_similarity",
+            message=message,
+            score=similarity,
+            expected=self.expected_text,
+            actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+            details={
+                "similarity": similarity,
+                "threshold": self.threshold,
+                "interpretation": interpretation,
+                "model": self.model_name,
+            },
+        )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> SemanticSimilarityAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "expected_text": "The expected output",
+                "threshold": 0.8,  # optional, default: 0.8
+                "model_name": "all-MiniLM-L6-v2"  # optional
+            }
+        """
+        if "expected_text" not in config:
+            raise ValueError("SemanticSimilarityAssertion requires 'expected_text' in config")
+
+        return cls(
+            expected_text=config["expected_text"],
+            threshold=config.get("threshold", 0.8),
+            model_name=config.get("model_name", "all-MiniLM-L6-v2"),
+        )
+
+    def __repr__(self) -> str:
+        return (
+            f"SemanticSimilarityAssertion("
+            f"expected_text={self.expected_text[:30]!r}..., "
+            f"threshold={self.threshold}, "
+            f"model_name={self.model_name!r})"
+        )
+
+    @classmethod
+    def clear_cache(cls) -> None:
+        """Clear the embedding cache. Useful for testing or memory management."""
+        cls._embedding_cache.clear()
+
+    @classmethod
+    def get_cache_size(cls) -> int:
+        """Get the number of cached embeddings."""
+        return len(cls._embedding_cache)

prela/evals/assertions/structural.py

@@ -0,0 +1,443 @@
+"""
+Structural assertions for text and data format validation.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from prela.core.span import Span
+from prela.evals.assertions.base import AssertionResult, BaseAssertion
+
+
+class ContainsAssertion(BaseAssertion):
+    """Assert that output contains specified text.
+
+    Example:
+        >>> assertion = ContainsAssertion(text="error", case_sensitive=False)
+        >>> result = assertion.evaluate(output="Error occurred", expected=None, trace=None)
+        >>> assert result.passed
+    """
+
+    def __init__(self, text: str, case_sensitive: bool = True):
+        """Initialize contains assertion.
+
+        Args:
+            text: Text that must be present in output
+            case_sensitive: Whether to perform case-sensitive matching
+        """
+        self.text = text
+        self.case_sensitive = case_sensitive
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output contains the specified text."""
+        output_str = str(output)
+        text = self.text
+
+        if not self.case_sensitive:
+            output_str = output_str.lower()
+            text = text.lower()
+
+        passed = text in output_str
+
+        if passed:
+            message = f"Output contains '{self.text}'"
+        else:
+            message = f"Output does not contain '{self.text}'"
+
+        return AssertionResult(
+            passed=passed,
+            assertion_type="contains",
+            message=message,
+            expected=self.text,
+            actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+            details={"case_sensitive": self.case_sensitive},
+        )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> ContainsAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "text": "required text",
+                "case_sensitive": true  # optional, default: true
+            }
+        """
+        if "text" not in config:
+            raise ValueError("ContainsAssertion requires 'text' in config")
+
+        return cls(
+            text=config["text"],
+            case_sensitive=config.get("case_sensitive", True),
+        )
+
+    def __repr__(self) -> str:
+        return f"ContainsAssertion(text={self.text!r}, case_sensitive={self.case_sensitive})"
+
+
+class NotContainsAssertion(BaseAssertion):
+    """Assert that output does NOT contain specified text.
+
+    Example:
+        >>> assertion = NotContainsAssertion(text="error")
+        >>> result = assertion.evaluate(output="Success!", expected=None, trace=None)
+        >>> assert result.passed
+    """
+
+    def __init__(self, text: str, case_sensitive: bool = True):
+        """Initialize not-contains assertion.
+
+        Args:
+            text: Text that must NOT be present in output
+            case_sensitive: Whether to perform case-sensitive matching
+        """
+        self.text = text
+        self.case_sensitive = case_sensitive
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output does not contain the specified text."""
+        output_str = str(output)
+        text = self.text
+
+        if not self.case_sensitive:
+            output_str = output_str.lower()
+            text = text.lower()
+
+        passed = text not in output_str
+
+        if passed:
+            message = f"Output correctly does not contain '{self.text}'"
+        else:
+            message = f"Output incorrectly contains '{self.text}'"
+
+        return AssertionResult(
+            passed=passed,
+            assertion_type="not_contains",
+            message=message,
+            expected=f"not containing '{self.text}'",
+            actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+            details={"case_sensitive": self.case_sensitive},
+        )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> NotContainsAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "text": "forbidden text",
+                "case_sensitive": true  # optional, default: true
+            }
+        """
+        if "text" not in config:
+            raise ValueError("NotContainsAssertion requires 'text' in config")
+
+        return cls(
+            text=config["text"],
+            case_sensitive=config.get("case_sensitive", True),
+        )
+
+    def __repr__(self) -> str:
+        return f"NotContainsAssertion(text={self.text!r}, case_sensitive={self.case_sensitive})"
+
+
+class RegexAssertion(BaseAssertion):
+    """Assert that output matches a regular expression pattern.
+
+    Example:
+        >>> assertion = RegexAssertion(pattern=r"\\d{3}-\\d{4}")
+        >>> result = assertion.evaluate(output="Call 555-1234", expected=None, trace=None)
+        >>> assert result.passed
+    """
+
+    def __init__(self, pattern: str, flags: int = 0):
+        """Initialize regex assertion.
+
+        Args:
+            pattern: Regular expression pattern to match
+            flags: Optional regex flags (e.g., re.IGNORECASE)
+        """
+        self.pattern = pattern
+        self.flags = flags
+        self._compiled = re.compile(pattern, flags)
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output matches the regex pattern."""
+        output_str = str(output)
+        match = self._compiled.search(output_str)
+        passed = match is not None
+
+        if passed:
+            matched_text = match.group(0) if match else ""
+            message = f"Output matches pattern '{self.pattern}' (matched: '{matched_text}')"
+            details = {
+                "matched_text": matched_text,
+                "match_start": match.start(),
+                "match_end": match.end(),
+            }
+        else:
+            message = f"Output does not match pattern '{self.pattern}'"
+            details = {}
+
+        return AssertionResult(
+            passed=passed,
+            assertion_type="regex",
+            message=message,
+            expected=self.pattern,
+            actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+            details=details,
+        )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> RegexAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "pattern": "\\d{3}-\\d{4}",
+                "flags": 2  # optional, e.g., re.IGNORECASE
+            }
+        """
+        if "pattern" not in config:
+            raise ValueError("RegexAssertion requires 'pattern' in config")
+
+        return cls(
+            pattern=config["pattern"],
+            flags=config.get("flags", 0),
+        )
+
+    def __repr__(self) -> str:
+        return f"RegexAssertion(pattern={self.pattern!r}, flags={self.flags})"
+
+
+class LengthAssertion(BaseAssertion):
+    """Assert that output length is within specified bounds.
+
+    Example:
+        >>> assertion = LengthAssertion(min_length=10, max_length=100)
+        >>> result = assertion.evaluate(output="Hello, world!", expected=None, trace=None)
+        >>> assert result.passed
+    """
+
+    def __init__(self, min_length: int | None = None, max_length: int | None = None):
+        """Initialize length assertion.
+
+        Args:
+            min_length: Minimum acceptable length (inclusive)
+            max_length: Maximum acceptable length (inclusive)
+
+        Raises:
+            ValueError: If both min_length and max_length are None
+        """
+        if min_length is None and max_length is None:
+            raise ValueError("At least one of min_length or max_length must be specified")
+
+        if min_length is not None and min_length < 0:
+            raise ValueError("min_length must be non-negative")
+
+        if max_length is not None and max_length < 0:
+            raise ValueError("max_length must be non-negative")
+
+        if min_length is not None and max_length is not None and min_length > max_length:
+            raise ValueError("min_length cannot be greater than max_length")
+
+        self.min_length = min_length
+        self.max_length = max_length
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output length is within bounds."""
+        output_str = str(output)
+        actual_length = len(output_str)
+
+        passed = True
+        reasons = []
+
+        if self.min_length is not None and actual_length < self.min_length:
+            passed = False
+            reasons.append(f"too short (< {self.min_length})")
+
+        if self.max_length is not None and actual_length > self.max_length:
+            passed = False
+            reasons.append(f"too long (> {self.max_length})")
+
+        if passed:
+            if self.min_length is not None and self.max_length is not None:
+                message = f"Output length {actual_length} is within bounds [{self.min_length}, {self.max_length}]"
+            elif self.min_length is not None:
+                message = f"Output length {actual_length} is >= {self.min_length}"
+            else:
+                message = f"Output length {actual_length} is <= {self.max_length}"
+        else:
+            message = f"Output length {actual_length} is {', '.join(reasons)}"
+
+        expected_desc = []
+        if self.min_length is not None:
+            expected_desc.append(f"min: {self.min_length}")
+        if self.max_length is not None:
+            expected_desc.append(f"max: {self.max_length}")
+
+        return AssertionResult(
+            passed=passed,
+            assertion_type="length",
+            message=message,
+            expected=", ".join(expected_desc),
+            actual=actual_length,
+            details={
+                "min_length": self.min_length,
+                "max_length": self.max_length,
+            },
+        )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> LengthAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "min_length": 10,  # optional
+                "max_length": 100  # optional
+            }
+        """
+        return cls(
+            min_length=config.get("min_length"),
+            max_length=config.get("max_length"),
+        )
+
+    def __repr__(self) -> str:
+        return f"LengthAssertion(min_length={self.min_length}, max_length={self.max_length})"
+
+
+class JSONValidAssertion(BaseAssertion):
+    """Assert that output is valid JSON, optionally matching a schema.
+
+    Example:
+        >>> assertion = JSONValidAssertion()
+        >>> result = assertion.evaluate(output='{"key": "value"}', expected=None, trace=None)
+        >>> assert result.passed
+    """
+
+    def __init__(self, schema: dict[str, Any] | None = None):
+        """Initialize JSON validation assertion.
+
+        Args:
+            schema: Optional JSON schema to validate against (using jsonschema library)
+        """
+        self.schema = schema
+
+        # Only import jsonschema if schema validation is requested
+        if schema is not None:
+            try:
+                import jsonschema
+                self._validator = jsonschema.Draft7Validator(schema)
+            except ImportError:
+                raise ImportError(
+                    "jsonschema library required for schema validation. "
+                    "Install with: pip install jsonschema"
+                )
+        else:
+            self._validator = None
+
+    def evaluate(
+        self,
+        output: Any,
+        expected: Any | None,
+        trace: list[Span] | None,
+    ) -> AssertionResult:
+        """Check if output is valid JSON and optionally matches schema."""
+        output_str = str(output)
+
+        # First, check if it's valid JSON
+        try:
+            parsed = json.loads(output_str)
+        except json.JSONDecodeError as e:
+            return AssertionResult(
+                passed=False,
+                assertion_type="json_valid",
+                message=f"Output is not valid JSON: {e.msg}",
+                expected="valid JSON",
+                actual=output_str[:100] + "..." if len(output_str) > 100 else output_str,
+                details={"error": str(e), "position": e.pos},
+            )
+
+        # If no schema, we're done
+        if self._validator is None:
+            return AssertionResult(
+                passed=True,
+                assertion_type="json_valid",
+                message="Output is valid JSON",
+                expected="valid JSON",
+                actual=parsed,
+                details={"type": type(parsed).__name__},
+            )
+
+        # Validate against schema
+        errors = list(self._validator.iter_errors(parsed))
+
+        if not errors:
+            return AssertionResult(
+                passed=True,
+                assertion_type="json_valid",
+                message="Output is valid JSON and matches schema",
+                expected="valid JSON matching schema",
+                actual=parsed,
+                details={"schema_valid": True},
+            )
+        else:
+            error_messages = [f"{e.json_path}: {e.message}" for e in errors[:3]]
+            if len(errors) > 3:
+                error_messages.append(f"... and {len(errors) - 3} more errors")
+
+            return AssertionResult(
+                passed=False,
+                assertion_type="json_valid",
+                message=f"Output is valid JSON but does not match schema: {'; '.join(error_messages)}",
+                expected="valid JSON matching schema",
+                actual=parsed,
+                details={
+                    "schema_valid": False,
+                    "error_count": len(errors),
+                    "errors": error_messages,
+                },
+            )
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> JSONValidAssertion:
+        """Create from configuration.
+
+        Config format:
+            {
+                "schema": {  # optional
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string"}
+                    }
+                }
+            }
+        """
+        return cls(schema=config.get("schema"))
+
+    def __repr__(self) -> str:
+        return f"JSONValidAssertion(schema={self.schema is not None})"