truthound 1.0.8__py3-none-any.whl

truthound/stores/expectations.py

@@ -0,0 +1,227 @@
+"""Expectation types for storage.
+
+This module defines the data structures for expectations (validation rules)
+that can be persisted and loaded from stores.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass
+class Expectation:
+    """A single validation expectation (rule).
+
+    An expectation defines what conditions a column or dataset should meet.
+    This is similar to Great Expectations' Expectation concept.
+
+    Attributes:
+        expectation_type: Type of expectation (e.g., "not_null", "in_range").
+        column: Column this expectation applies to (None for table-level).
+        kwargs: Parameters for the expectation.
+        meta: Additional metadata about the expectation.
+        enabled: Whether this expectation is active.
+        mostly: Fraction of rows that must pass (0.0 to 1.0).
+    """
+
+    expectation_type: str
+    column: str | None = None
+    kwargs: dict[str, Any] = field(default_factory=dict)
+    meta: dict[str, Any] = field(default_factory=dict)
+    enabled: bool = True
+    mostly: float = 1.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "expectation_type": self.expectation_type,
+            "column": self.column,
+            "kwargs": self.kwargs,
+            "meta": self.meta,
+            "enabled": self.enabled,
+            "mostly": self.mostly,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Expectation":
+        """Create from dictionary."""
+        return cls(
+            expectation_type=data["expectation_type"],
+            column=data.get("column"),
+            kwargs=data.get("kwargs", {}),
+            meta=data.get("meta", {}),
+            enabled=data.get("enabled", True),
+            mostly=data.get("mostly", 1.0),
+        )
+
+    def __str__(self) -> str:
+        """Human-readable representation."""
+        if self.column:
+            return f"{self.expectation_type}(column={self.column})"
+        return f"{self.expectation_type}"
+
+
+@dataclass
+class ExpectationSuite:
+    """A collection of expectations for a data asset.
+
+    An expectation suite groups related expectations together and provides
+    metadata about when and how they should be applied.
+
+    Attributes:
+        name: Unique name for this suite.
+        data_asset: Name of the data asset this suite validates.
+        expectations: List of expectations in this suite.
+        meta: Additional metadata about the suite.
+        created_at: When the suite was created.
+        updated_at: When the suite was last updated.
+        tags: Key-value tags for filtering/grouping.
+    """
+
+    name: str
+    data_asset: str
+    expectations: list[Expectation] = field(default_factory=list)
+    meta: dict[str, Any] = field(default_factory=dict)
+    created_at: datetime = field(default_factory=datetime.now)
+    updated_at: datetime = field(default_factory=datetime.now)
+    tags: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def id(self) -> str:
+        """Unique identifier for this suite."""
+        return self.name
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "name": self.name,
+            "data_asset": self.data_asset,
+            "expectations": [e.to_dict() for e in self.expectations],
+            "meta": self.meta,
+            "created_at": self.created_at.isoformat(),
+            "updated_at": self.updated_at.isoformat(),
+            "tags": self.tags,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ExpectationSuite":
+        """Create from dictionary."""
+        return cls(
+            name=data["name"],
+            data_asset=data["data_asset"],
+            expectations=[Expectation.from_dict(e) for e in data.get("expectations", [])],
+            meta=data.get("meta", {}),
+            created_at=datetime.fromisoformat(data["created_at"]) if "created_at" in data else datetime.now(),
+            updated_at=datetime.fromisoformat(data["updated_at"]) if "updated_at" in data else datetime.now(),
+            tags=data.get("tags", {}),
+        )
+
+    def add_expectation(self, expectation: Expectation) -> None:
+        """Add an expectation to the suite.
+
+        Args:
+            expectation: The expectation to add.
+        """
+        self.expectations.append(expectation)
+        self.updated_at = datetime.now()
+
+    def remove_expectation(self, index: int) -> Expectation:
+        """Remove an expectation by index.
+
+        Args:
+            index: Index of the expectation to remove.
+
+        Returns:
+            The removed expectation.
+        """
+        expectation = self.expectations.pop(index)
+        self.updated_at = datetime.now()
+        return expectation
+
+    def get_expectations_for_column(self, column: str) -> list[Expectation]:
+        """Get all expectations for a specific column.
+
+        Args:
+            column: Column name to filter by.
+
+        Returns:
+            List of expectations for the column.
+        """
+        return [e for e in self.expectations if e.column == column]
+
+    def get_table_expectations(self) -> list[Expectation]:
+        """Get all table-level expectations (not column-specific).
+
+        Returns:
+            List of table-level expectations.
+        """
+        return [e for e in self.expectations if e.column is None]
+
+    def get_enabled_expectations(self) -> list[Expectation]:
+        """Get all enabled expectations.
+
+        Returns:
+            List of enabled expectations.
+        """
+        return [e for e in self.expectations if e.enabled]
+
+    @property
+    def expectation_count(self) -> int:
+        """Get the total number of expectations."""
+        return len(self.expectations)
+
+    @property
+    def enabled_count(self) -> int:
+        """Get the number of enabled expectations."""
+        return len(self.get_enabled_expectations())
+
+    def summary(self) -> str:
+        """Get a human-readable summary of the suite."""
+        columns = {e.column for e in self.expectations if e.column}
+        lines = [
+            f"Expectation Suite: {self.name}",
+            f"Data Asset: {self.data_asset}",
+            f"Expectations: {self.expectation_count} ({self.enabled_count} enabled)",
+            f"Columns: {len(columns)}",
+            f"Updated: {self.updated_at.isoformat()}",
+        ]
+        return "\n".join(lines)
+
+    @classmethod
+    def create(
+        cls,
+        name: str,
+        data_asset: str,
+        expectations: list[dict[str, Any]] | None = None,
+        **kwargs: Any,
+    ) -> "ExpectationSuite":
+        """Create a new expectation suite with a convenient API.
+
+        Args:
+            name: Name for the suite.
+            data_asset: Data asset this suite validates.
+            expectations: Optional list of expectation dicts.
+            **kwargs: Additional suite attributes.
+
+        Returns:
+            New ExpectationSuite instance.
+
+        Example:
+            >>> suite = ExpectationSuite.create(
+            ...     name="customer_suite",
+            ...     data_asset="customers.csv",
+            ...     expectations=[
+            ...         {"expectation_type": "not_null", "column": "email"},
+            ...         {"expectation_type": "unique", "column": "id"},
+            ...     ]
+            ... )
+        """
+        suite = cls(name=name, data_asset=data_asset, **kwargs)
+        if expectations:
+            for exp_dict in expectations:
+                suite.add_expectation(Expectation.from_dict(exp_dict))
+        return suite

truthound/stores/factory.py

@@ -0,0 +1,246 @@
+"""Factory functions for creating stores.
+
+This module provides a registry-based factory pattern for creating store
+instances. New store backends can be registered at runtime.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, TypeVar
+
+from truthound.stores.base import BaseStore, StoreConfig, StoreError
+
+# Type for store constructor functions
+StoreConstructor = Callable[..., BaseStore[Any, Any]]
+
+# Registry of store constructors
+_store_registry: dict[str, StoreConstructor] = {}
+
+
+def register_store(name: str) -> Callable[[StoreConstructor], StoreConstructor]:
+    """Decorator to register a store backend.
+
+    Args:
+        name: Name to register the store under.
+
+    Returns:
+        Decorator function.
+
+    Example:
+        >>> @register_store("my_store")
+        ... class MyStore(BaseStore):
+        ...     pass
+    """
+
+    def decorator(cls: StoreConstructor) -> StoreConstructor:
+        _store_registry[name] = cls
+        return cls
+
+    return decorator
+
+
+def get_store(backend: str, **kwargs: Any) -> BaseStore[Any, Any]:
+    """Create a store instance for the specified backend.
+
+    This is the primary entry point for creating stores. It handles
+    lazy loading of backend modules and provides a uniform interface.
+
+    Args:
+        backend: Name of the store backend to use. Options:
+            - "filesystem": Local filesystem storage (default)
+            - "s3": AWS S3 storage (requires boto3)
+            - "gcs": Google Cloud Storage (requires google-cloud-storage)
+            - "database": SQL database storage (requires sqlalchemy)
+            - "memory": In-memory storage (for testing)
+        **kwargs: Backend-specific configuration options.
+
+    Returns:
+        Configured store instance.
+
+    Raises:
+        StoreError: If the backend is not available or configuration fails.
+
+    Example:
+        >>> # Filesystem store
+        >>> store = get_store("filesystem", base_path=".truthound/results")
+        >>>
+        >>> # S3 store
+        >>> store = get_store("s3", bucket="my-bucket", prefix="validations/")
+        >>>
+        >>> # Memory store (for testing)
+        >>> store = get_store("memory")
+    """
+    # Normalize backend name
+    backend = backend.lower().strip()
+
+    # Check if already registered
+    if backend in _store_registry:
+        return _store_registry[backend](**kwargs)
+
+    # Lazy load built-in backends
+    if backend == "filesystem":
+        from truthound.stores.backends.filesystem import FileSystemStore
+
+        return FileSystemStore(**kwargs)
+
+    elif backend == "memory":
+        from truthound.stores.backends.memory import MemoryStore
+
+        return MemoryStore(**kwargs)
+
+    elif backend == "s3":
+        try:
+            import boto3  # noqa: F401
+        except ImportError:
+            raise StoreError(
+                "S3 backend requires boto3. Install with: pip install truthound[s3]"
+            )
+        from truthound.stores.backends.s3 import S3Store
+
+        return S3Store(**kwargs)
+
+    elif backend == "gcs":
+        try:
+            import google.cloud.storage  # noqa: F401
+        except ImportError:
+            raise StoreError(
+                "GCS backend requires google-cloud-storage. "
+                "Install with: pip install truthound[gcs]"
+            )
+        from truthound.stores.backends.gcs import GCSStore
+
+        return GCSStore(**kwargs)
+
+    elif backend in ("database", "db", "sql"):
+        try:
+            import sqlalchemy  # noqa: F401
+        except ImportError:
+            raise StoreError(
+                "Database backend requires sqlalchemy. "
+                "Install with: pip install truthound[database]"
+            )
+        from truthound.stores.backends.database import DatabaseStore
+
+        return DatabaseStore(**kwargs)
+
+    elif backend in ("azure", "azure_blob", "azureblob"):
+        try:
+            import azure.storage.blob  # noqa: F401
+        except ImportError:
+            raise StoreError(
+                "Azure Blob backend requires azure-storage-blob. "
+                "Install with: pip install truthound[azure]"
+            )
+        from truthound.stores.backends.azure_blob import AzureBlobStore
+
+        return AzureBlobStore(**kwargs)
+
+    else:
+        available = list(_store_registry.keys()) + [
+            "filesystem",
+            "memory",
+            "s3",
+            "gcs",
+            "database",
+            "azure",
+        ]
+        raise StoreError(
+            f"Unknown store backend: {backend}. "
+            f"Available backends: {', '.join(sorted(set(available)))}"
+        )
+
+
+def list_available_backends() -> list[str]:
+    """List all available store backends.
+
+    Returns:
+        List of backend names that can be used with get_store().
+    """
+    # Built-in backends always available
+    backends = ["filesystem", "memory"]
+
+    # Check optional backends
+    try:
+        import boto3
+
+        backends.append("s3")
+    except ImportError:
+        pass
+
+    try:
+        import google.cloud.storage
+
+        backends.append("gcs")
+    except ImportError:
+        pass
+
+    try:
+        import sqlalchemy
+
+        backends.append("database")
+    except ImportError:
+        pass
+
+    try:
+        import azure.storage.blob
+
+        backends.append("azure")
+    except ImportError:
+        pass
+
+    # Add registered backends
+    backends.extend(_store_registry.keys())
+
+    return sorted(set(backends))
+
+
+def is_backend_available(backend: str) -> bool:
+    """Check if a backend is available.
+
+    Args:
+        backend: Name of the backend to check.
+
+    Returns:
+        True if the backend is available, False otherwise.
+    """
+    backend = backend.lower().strip()
+
+    if backend in ("filesystem", "memory"):
+        return True
+
+    if backend in _store_registry:
+        return True
+
+    if backend == "s3":
+        try:
+            import boto3
+
+            return True
+        except ImportError:
+            return False
+
+    if backend == "gcs":
+        try:
+            import google.cloud.storage
+
+            return True
+        except ImportError:
+            return False
+
+    if backend in ("database", "db", "sql"):
+        try:
+            import sqlalchemy
+
+            return True
+        except ImportError:
+            return False
+
+    if backend in ("azure", "azure_blob", "azureblob"):
+        try:
+            import azure.storage.blob
+
+            return True
+        except ImportError:
+            return False
+
+    return False

truthound/stores/migration/__init__.py

@@ -0,0 +1,75 @@
+"""Schema migration system for validation stores.
+
+This module provides a schema migration layer that handles versioning
+of data formats and automatic migration between schema versions.
+
+Example:
+    >>> from truthound.stores import get_store
+    >>> from truthound.stores.migration import (
+    ...     MigratableStore,
+    ...     MigrationConfig,
+    ...     SchemaMigration,
+    ...     MigrationRegistry,
+    ... )
+    >>>
+    >>> # Define migrations
+    >>> registry = MigrationRegistry()
+    >>>
+    >>> @registry.register("1.0.0", "2.0.0")
+    ... def migrate_v1_to_v2(data: dict) -> dict:
+    ...     # Add new field
+    ...     data["new_field"] = data.get("old_field", "")
+    ...     return data
+    >>>
+    >>> @registry.register("2.0.0", "3.0.0")
+    ... def migrate_v2_to_v3(data: dict) -> dict:
+    ...     # Rename field
+    ...     data["renamed"] = data.pop("old_name", None)
+    ...     return data
+    >>>
+    >>> # Create migratable store
+    >>> base_store = get_store("filesystem", base_path=".truthound/results")
+    >>> store = MigratableStore(
+    ...     base_store,
+    ...     MigrationConfig(
+    ...         current_version="3.0.0",
+    ...         registry=registry,
+    ...         auto_migrate=True,
+    ...     ),
+    ... )
+    >>>
+    >>> # Old data is automatically migrated on read
+    >>> result = store.get(run_id)
+"""
+
+from truthound.stores.migration.base import (
+    MigrationConfig,
+    MigrationInfo,
+    MigrationResult,
+    SchemaMigration,
+    SchemaVersion,
+    MigrationError,
+    IncompatibleVersionError,
+    MigrationPathNotFoundError,
+)
+from truthound.stores.migration.registry import MigrationRegistry
+from truthound.stores.migration.store import MigratableStore
+from truthound.stores.migration.manager import MigrationManager
+
+__all__ = [
+    # Base types
+    "MigrationConfig",
+    "MigrationInfo",
+    "MigrationResult",
+    "SchemaMigration",
+    "SchemaVersion",
+    "MigrationError",
+    "IncompatibleVersionError",
+    "MigrationPathNotFoundError",
+    # Registry
+    "MigrationRegistry",
+    # Store wrapper
+    "MigratableStore",
+    # Manager
+    "MigrationManager",
+]