prela 0.1.0__py3-none-any.whl

prela/evals/suite.py

@@ -0,0 +1,316 @@
+"""Eval suite for organizing and managing test cases.
+
+This module provides the EvalSuite class for organizing multiple eval cases,
+with support for YAML serialization, setup/teardown hooks, and default assertions.
+
+Example:
+    >>> from prela.evals import EvalSuite, EvalCase, EvalInput, EvalExpected
+    >>> suite = EvalSuite(
+    ...     name="RAG Quality Suite",
+    ...     description="Tests for RAG pipeline quality",
+    ...     cases=[
+    ...         EvalCase(
+    ...             id="test_basic_qa",
+    ...             name="Basic QA test",
+    ...             input=EvalInput(query="What is 2+2?"),
+    ...             expected=EvalExpected(contains=["4"])
+    ...         )
+    ...     ]
+    ... )
+    >>> suite.to_yaml("suite.yaml")
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable
+
+try:
+    import yaml
+
+    YAML_AVAILABLE = True
+except ImportError:
+    YAML_AVAILABLE = False
+
+from prela.evals.case import EvalCase
+
+
+@dataclass
+class EvalSuite:
+    """Collection of eval cases with shared configuration.
+
+    An eval suite organizes multiple test cases with:
+    - Shared setup/teardown hooks
+    - Default assertions applied to all cases
+    - YAML serialization for easy configuration
+    - Tagging and filtering capabilities
+
+    Attributes:
+        name: Suite name (e.g., "RAG Quality Suite")
+        description: Human-readable description of what this suite tests
+        cases: List of eval cases in this suite
+        default_assertions: Assertions applied to all cases (unless overridden)
+        setup: Callable run before executing the suite (e.g., start services)
+        teardown: Callable run after executing the suite (e.g., cleanup)
+        metadata: Additional metadata for the suite
+
+    Example:
+        >>> suite = EvalSuite(
+        ...     name="RAG Quality Suite",
+        ...     description="Tests for RAG pipeline quality",
+        ...     cases=[
+        ...         EvalCase(
+        ...             id="test_basic_qa",
+        ...             name="Basic factual question",
+        ...             input=EvalInput(query="What is the capital of France?"),
+        ...             expected=EvalExpected(contains=["Paris"])
+        ...         )
+        ...     ],
+        ...     default_assertions=[
+        ...         {"type": "latency", "max_ms": 5000},
+        ...         {"type": "no_errors"}
+        ...     ]
+        ... )
+    """
+
+    name: str
+    description: str = ""
+    cases: list[EvalCase] = field(default_factory=list)
+    default_assertions: list[dict[str, Any]] | None = None
+    setup: Callable[[], None] | None = None
+    teardown: Callable[[], None] | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        """Validate suite configuration."""
+        if not self.name:
+            raise ValueError("EvalSuite must have a non-empty 'name'")
+
+    def add_case(self, case: EvalCase) -> None:
+        """Add a test case to the suite.
+
+        Args:
+            case: Eval case to add
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite")
+            >>> case = EvalCase(
+            ...     id="test_1",
+            ...     name="Test",
+            ...     input=EvalInput(query="Hello"),
+            ...     expected=EvalExpected(contains=["Hi"])
+            ... )
+            >>> suite.add_case(case)
+        """
+        self.cases.append(case)
+
+    def get_case(self, case_id: str) -> EvalCase | None:
+        """Get a test case by ID.
+
+        Args:
+            case_id: ID of the test case to retrieve
+
+        Returns:
+            EvalCase if found, None otherwise
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[...])
+            >>> case = suite.get_case("test_basic_qa")
+        """
+        for case in self.cases:
+            if case.id == case_id:
+                return case
+        return None
+
+    def filter_by_tags(self, tags: list[str]) -> list[EvalCase]:
+        """Filter test cases by tags.
+
+        Returns cases that have ALL specified tags.
+
+        Args:
+            tags: List of tags to filter by
+
+        Returns:
+            List of matching test cases
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[...])
+            >>> qa_cases = suite.filter_by_tags(["qa"])
+            >>> geography_qa = suite.filter_by_tags(["qa", "geography"])
+        """
+        return [case for case in self.cases if all(tag in case.tags for tag in tags)]
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalSuite:
+        """Create EvalSuite from dictionary.
+
+        Args:
+            data: Dictionary with suite specification
+
+        Returns:
+            EvalSuite instance
+
+        Example:
+            >>> data = {
+            ...     "name": "My Suite",
+            ...     "description": "Test suite",
+            ...     "cases": [
+            ...         {
+            ...             "id": "test_1",
+            ...             "name": "Test",
+            ...             "input": {"query": "Hello"},
+            ...             "expected": {"contains": ["Hi"]}
+            ...         }
+            ...     ]
+            ... }
+            >>> suite = EvalSuite.from_dict(data)
+        """
+        # Parse cases
+        cases_data = data.get("cases", [])
+        cases = [EvalCase.from_dict(case_data) for case_data in cases_data]
+
+        return cls(
+            name=data["name"],
+            description=data.get("description", ""),
+            cases=cases,
+            default_assertions=data.get("default_assertions"),
+            metadata=data.get("metadata", {}),
+            # Note: setup/teardown can't be serialized, only set programmatically
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization.
+
+        Returns:
+            Dictionary representation of the suite.
+
+        Note:
+            setup and teardown callables are not serialized.
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[...])
+            >>> data = suite.to_dict()
+            >>> data["name"]
+            'My Suite'
+        """
+        result: dict[str, Any] = {
+            "name": self.name,
+        }
+
+        if self.description:
+            result["description"] = self.description
+
+        if len(self.cases) > 0:
+            result["cases"] = [case.to_dict() for case in self.cases]
+
+        if self.default_assertions is not None and len(self.default_assertions) > 0:
+            result["default_assertions"] = self.default_assertions
+
+        if len(self.metadata) > 0:
+            result["metadata"] = self.metadata
+
+        return result
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> EvalSuite:
+        """Load eval suite from YAML file.
+
+        Args:
+            path: Path to YAML file
+
+        Returns:
+            EvalSuite instance
+
+        Raises:
+            ImportError: If PyYAML is not installed
+            FileNotFoundError: If file doesn't exist
+            yaml.YAMLError: If YAML parsing fails
+
+        Example:
+            >>> suite = EvalSuite.from_yaml("tests/suite.yaml")
+        """
+        if not YAML_AVAILABLE:
+            raise ImportError(
+                "PyYAML is required for YAML support. "
+                "Install with: pip install pyyaml"
+            )
+
+        path = Path(path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        with open(path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+
+        return cls.from_dict(data)
+
+    def to_yaml(self, path: str | Path) -> None:
+        """Save eval suite to YAML file.
+
+        Args:
+            path: Path to save YAML file
+
+        Raises:
+            ImportError: If PyYAML is not installed
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[...])
+            >>> suite.to_yaml("suite.yaml")
+        """
+        if not YAML_AVAILABLE:
+            raise ImportError(
+                "PyYAML is required for YAML support. "
+                "Install with: pip install pyyaml"
+            )
+
+        path = Path(path)
+
+        # Create parent directory if it doesn't exist
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        data = self.to_dict()
+
+        with open(path, "w", encoding="utf-8") as f:
+            yaml.dump(
+                data,
+                f,
+                default_flow_style=False,
+                sort_keys=False,
+                allow_unicode=True,
+            )
+
+    def __len__(self) -> int:
+        """Return number of test cases in suite.
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[case1, case2])
+            >>> len(suite)
+            2
+        """
+        return len(self.cases)
+
+    def __iter__(self):
+        """Iterate over test cases.
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[case1, case2])
+            >>> for case in suite:
+            ...     print(case.name)
+        """
+        return iter(self.cases)
+
+    def __getitem__(self, index: int) -> EvalCase:
+        """Get test case by index.
+
+        Args:
+            index: Index of the test case
+
+        Returns:
+            EvalCase at the specified index
+
+        Example:
+            >>> suite = EvalSuite(name="My Suite", cases=[case1, case2])
+            >>> first_case = suite[0]
+        """
+        return self.cases[index]

prela/exporters/__init__.py

@@ -0,0 +1,27 @@
+"""Exporters for sending spans to external systems."""
+
+from prela.exporters.base import BaseExporter, BatchExporter, ExportResult
+from prela.exporters.console import ConsoleExporter
+from prela.exporters.file import FileExporter
+from prela.exporters.http import HTTPExporter
+from prela.exporters.multi import MultiExporter
+
+# OTLP exporter requires optional dependency
+try:
+    from prela.exporters.otlp import OTLPExporter
+
+    OTLP_AVAILABLE = True
+except ImportError:
+    OTLP_AVAILABLE = False
+    OTLPExporter = None  # type: ignore
+
+__all__ = [
+    "BaseExporter",
+    "BatchExporter",
+    "ExportResult",
+    "ConsoleExporter",
+    "FileExporter",
+    "HTTPExporter",
+    "MultiExporter",
+    "OTLPExporter",
+]

prela/exporters/base.py

@@ -0,0 +1,189 @@
+"""Base classes for span exporters.
+
+This module provides abstract base classes for implementing span exporters.
+Exporters are responsible for sending completed spans to external systems
+like observability platforms, databases, or files.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any
+
+from prela.core.span import Span
+
+logger = logging.getLogger(__name__)
+
+
+class ExportResult(Enum):
+    """Result of an export operation."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    RETRY = "retry"
+
+
+class BaseExporter(ABC):
+    """Abstract base class for span exporters.
+
+    Exporters are responsible for sending spans to external systems.
+    Implementations must handle serialization, network requests, and error handling.
+    """
+
+    @abstractmethod
+    def export(self, spans: list[Span]) -> None:
+        """Export a batch of spans.
+
+        Args:
+            spans: List of spans to export
+
+        Raises:
+            Exception: If export fails and should not be retried
+        """
+        pass
+
+    @abstractmethod
+    def shutdown(self) -> None:
+        """Shutdown the exporter and flush any pending data.
+
+        This method should be called before the application exits to ensure
+        all spans are properly exported.
+        """
+        pass
+
+
+class BatchExporter(BaseExporter):
+    """Base class for exporters that batch spans with retry logic.
+
+    This class handles common batching concerns:
+    - Retry with exponential backoff
+    - Timeout handling
+    - Error logging
+
+    Subclasses only need to implement _do_export() to define how spans
+    are actually sent to the backend.
+    """
+
+    def __init__(
+        self,
+        max_retries: int = 3,
+        initial_backoff_ms: float = 100.0,
+        max_backoff_ms: float = 10000.0,
+        timeout_ms: float = 30000.0,
+    ) -> None:
+        """Initialize the batch exporter.
+
+        Args:
+            max_retries: Maximum number of retry attempts
+            initial_backoff_ms: Initial backoff delay in milliseconds
+            max_backoff_ms: Maximum backoff delay in milliseconds
+            timeout_ms: Timeout for export operation in milliseconds
+        """
+        self.max_retries = max_retries
+        self.initial_backoff_ms = initial_backoff_ms
+        self.max_backoff_ms = max_backoff_ms
+        self.timeout_ms = timeout_ms
+        self._shutdown = False
+
+    @abstractmethod
+    def _do_export(self, spans: list[Span]) -> ExportResult:
+        """Perform the actual export operation.
+
+        This method should be implemented by subclasses to define how spans
+        are sent to the backend system.
+
+        Args:
+            spans: List of spans to export
+
+        Returns:
+            ExportResult indicating success, failure, or retry needed
+        """
+        pass
+
+    def export(self, spans: list[Span]) -> None:
+        """Export spans with retry logic.
+
+        Args:
+            spans: List of spans to export
+
+        Raises:
+            RuntimeError: If exporter is shutdown
+            Exception: If export fails after all retries
+        """
+        if self._shutdown:
+            raise RuntimeError("Cannot export: exporter is shutdown")
+
+        if not spans:
+            return
+
+        start_time = time.perf_counter()
+        attempt = 0
+        backoff_ms = self.initial_backoff_ms
+
+        while attempt <= self.max_retries:
+            # Check timeout
+            elapsed_ms = (time.perf_counter() - start_time) * 1000
+            if elapsed_ms >= self.timeout_ms:
+                raise TimeoutError(
+                    f"Export timeout after {elapsed_ms:.2f}ms " f"(limit: {self.timeout_ms}ms)"
+                )
+
+            try:
+                result = self._do_export(spans)
+
+                if result == ExportResult.SUCCESS:
+                    logger.debug(
+                        "Successfully exported %d spans on attempt %d",
+                        len(spans),
+                        attempt + 1,
+                    )
+                    return
+
+                if result == ExportResult.FAILURE:
+                    raise Exception(f"Export failed permanently on attempt {attempt + 1}")
+
+                # result == ExportResult.RETRY
+                if attempt < self.max_retries:
+                    logger.warning(
+                        "Export needs retry (attempt %d/%d), backing off %.2fms",
+                        attempt + 1,
+                        self.max_retries + 1,
+                        backoff_ms,
+                    )
+                    time.sleep(backoff_ms / 1000)
+                    backoff_ms = min(backoff_ms * 2, self.max_backoff_ms)
+
+            except Exception as e:
+                if attempt >= self.max_retries:
+                    logger.error(
+                        "Export failed after %d attempts: %s",
+                        attempt + 1,
+                        str(e),
+                    )
+                    raise
+
+                logger.warning(
+                    "Export failed (attempt %d/%d): %s, backing off %.2fms",
+                    attempt + 1,
+                    self.max_retries + 1,
+                    str(e),
+                    backoff_ms,
+                )
+                time.sleep(backoff_ms / 1000)
+                backoff_ms = min(backoff_ms * 2, self.max_backoff_ms)
+
+            attempt += 1
+
+        raise Exception(f"Export failed after {self.max_retries + 1} attempts")
+
+    def shutdown(self) -> None:
+        """Shutdown the exporter.
+
+        Subclasses can override this to implement custom shutdown logic
+        like flushing buffers or closing connections.
+        """
+        self._shutdown = True
+        logger.info("Exporter shutdown")