odibi 2.5.0__py3-none-any.whl

odibi/story/themes.py

@@ -0,0 +1,216 @@
+"""
+Theme System
+============
+
+Customizable themes for story rendering with branding support.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+
+
+@dataclass
+class StoryTheme:
+    """
+    Story theme configuration.
+
+    Defines colors, typography, branding, and layout options for
+    rendered stories (HTML).
+    """
+
+    name: str
+
+    # Colors
+    primary_color: str = "#0066cc"
+    success_color: str = "#28a745"
+    error_color: str = "#dc3545"
+    warning_color: str = "#ffc107"
+    bg_color: str = "#ffffff"
+    text_color: str = "#333333"
+    border_color: str = "#dddddd"
+    code_bg: str = "#f5f5f5"
+
+    # Typography
+    font_family: str = (
+        "system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif"
+    )
+    heading_font: str = "inherit"
+    code_font: str = "Consolas, Monaco, 'Courier New', monospace"
+    font_size: str = "16px"
+
+    # Branding
+    logo_url: Optional[str] = None
+    company_name: Optional[str] = None
+    footer_text: Optional[str] = None
+
+    # Layout
+    max_width: str = "1200px"
+    sidebar: bool = False
+
+    # Custom CSS
+    custom_css: Optional[str] = None
+
+    def to_css_vars(self) -> Dict[str, str]:
+        """
+        Convert theme to CSS variables.
+
+        Returns:
+            Dictionary of CSS variable names and values
+        """
+        return {
+            "--primary-color": self.primary_color,
+            "--success-color": self.success_color,
+            "--error-color": self.error_color,
+            "--warning-color": self.warning_color,
+            "--bg-color": self.bg_color,
+            "--text-color": self.text_color,
+            "--border-color": self.border_color,
+            "--code-bg": self.code_bg,
+            "--font-family": self.font_family,
+            "--heading-font": self.heading_font,
+            "--code-font": self.code_font,
+            "--font-size": self.font_size,
+            "--max-width": self.max_width,
+        }
+
+    def to_css_string(self) -> str:
+        """
+        Generate CSS string from theme.
+
+        Returns:
+            CSS string with :root variables
+        """
+        lines = [":root {"]
+        for var_name, var_value in self.to_css_vars().items():
+            lines.append(f"    {var_name}: {var_value};")
+        lines.append("}")
+
+        if self.custom_css:
+            lines.append("")
+            lines.append(self.custom_css)
+
+        return "\n".join(lines)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "StoryTheme":
+        """
+        Create theme from dictionary.
+
+        Args:
+            data: Theme configuration dictionary
+
+        Returns:
+            StoryTheme instance
+        """
+        return cls(**data)
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "StoryTheme":
+        """
+        Load theme from YAML file.
+
+        Args:
+            path: Path to YAML theme file
+
+        Returns:
+            StoryTheme instance
+        """
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        return cls.from_dict(data)
+
+
+# Built-in Themes
+# ===============
+
+DEFAULT_THEME = StoryTheme(
+    name="default",
+)
+
+CORPORATE_THEME = StoryTheme(
+    name="corporate",
+    primary_color="#003366",
+    success_color="#2e7d32",
+    error_color="#c62828",
+    font_family="Arial, Helvetica, sans-serif",
+    heading_font="Georgia, 'Times New Roman', serif",
+    font_size="15px",
+)
+
+DARK_THEME = StoryTheme(
+    name="dark",
+    primary_color="#00bfff",
+    success_color="#4caf50",
+    error_color="#f44336",
+    warning_color="#ffb300",
+    bg_color="#1e1e1e",
+    text_color="#e0e0e0",
+    border_color="#444444",
+    code_bg="#2d2d2d",
+    custom_css="""
+    body { background: #121212; }
+    .container { background: #1e1e1e; }
+    .node-header { background: #2d2d2d; }
+    .summary { background: #2d2d2d; }
+    """,
+)
+
+MINIMAL_THEME = StoryTheme(
+    name="minimal",
+    primary_color="#000000",
+    success_color="#006600",
+    error_color="#cc0000",
+    warning_color="#ff9900",
+    font_family="'Helvetica Neue', Helvetica, Arial, sans-serif",
+    heading_font="'Helvetica Neue', Helvetica, Arial, sans-serif",
+    font_size="14px",
+    max_width="900px",
+)
+
+# Theme registry
+BUILTIN_THEMES = {
+    "default": DEFAULT_THEME,
+    "corporate": CORPORATE_THEME,
+    "dark": DARK_THEME,
+    "minimal": MINIMAL_THEME,
+}
+
+
+def get_theme(name: str) -> StoryTheme:
+    """
+    Get theme by name.
+
+    Args:
+        name: Theme name or path to YAML theme file
+
+    Returns:
+        StoryTheme instance
+
+    Raises:
+        ValueError: If theme not found
+    """
+    # Check if it's a file path
+    if Path(name).exists():
+        return StoryTheme.from_yaml(name)
+
+    # Check built-in themes
+    if name.lower() in BUILTIN_THEMES:
+        return BUILTIN_THEMES[name.lower()]
+
+    raise ValueError(
+        f"Theme '{name}' not found. Available themes: {', '.join(BUILTIN_THEMES.keys())}"
+    )
+
+
+def list_themes() -> Dict[str, StoryTheme]:
+    """
+    List all available built-in themes.
+
+    Returns:
+        Dictionary of theme name -> StoryTheme
+    """
+    return BUILTIN_THEMES.copy()

odibi/testing/__init__.py

@@ -0,0 +1,13 @@
+"""Testing utilities for Odibi."""
+
+from .assertions import assert_frame_equal, assert_schema_equal
+from .fixtures import generate_sample_data, temp_directory
+
+__version__ = "1.3.0-alpha.1"
+
+__all__ = [
+    "temp_directory",
+    "generate_sample_data",
+    "assert_frame_equal",
+    "assert_schema_equal",
+]

odibi/testing/assertions.py

@@ -0,0 +1,75 @@
+"""
+Testing Assertions
+==================
+
+Helpers for asserting DataFrame equality and properties.
+"""
+
+from typing import Any, List
+
+import pandas as pd
+
+
+def assert_frame_equal(
+    left: Any,
+    right: Any,
+    check_dtype: bool = True,
+    check_exact: bool = False,
+    atol: float = 1e-8,
+    rtol: float = 1e-5,
+) -> None:
+    """
+    Assert that two DataFrames are equal.
+    Supports both Pandas and Spark DataFrames.
+
+    Args:
+        left: First DataFrame
+        right: Second DataFrame
+        check_dtype: Whether to check data types
+        check_exact: Whether to compare numbers exactly
+        atol: Absolute tolerance
+        rtol: Relative tolerance
+    """
+    # Convert Spark to Pandas for comparison if needed
+    left_pdf = _to_pandas(left)
+    right_pdf = _to_pandas(right)
+
+    # Sort by first column to ensure order doesn't matter (Spark is unordered)
+    if not left_pdf.empty and not right_pdf.empty:
+        sort_col = left_pdf.columns[0]
+        left_pdf = left_pdf.sort_values(sort_col).reset_index(drop=True)
+        right_pdf = right_pdf.sort_values(sort_col).reset_index(drop=True)
+
+    pd.testing.assert_frame_equal(
+        left_pdf, right_pdf, check_dtype=check_dtype, check_exact=check_exact, atol=atol, rtol=rtol
+    )
+
+
+def assert_schema_equal(left: Any, right: Any) -> None:
+    """
+    Assert that two DataFrames have the same schema (column names and types).
+    """
+    # Simplified check for column names
+    left_cols = sorted(_get_columns(left))
+    right_cols = sorted(_get_columns(right))
+
+    assert left_cols == right_cols, f"Schema mismatch: {left_cols} != {right_cols}"
+
+
+def _to_pandas(df: Any) -> pd.DataFrame:
+    """Convert to Pandas DataFrame if not already."""
+    if isinstance(df, pd.DataFrame):
+        return df
+
+    # Assume Spark DataFrame
+    try:
+        return df.toPandas()
+    except AttributeError:
+        raise TypeError(f"Expected DataFrame, got {type(df)}")
+
+
+def _get_columns(df: Any) -> List[str]:
+    """Get column names."""
+    if isinstance(df, pd.DataFrame):
+        return list(df.columns)
+    return list(df.columns)

odibi/testing/fixtures.py

@@ -0,0 +1,85 @@
+"""
+Testing Fixtures
+================
+
+Reusable fixtures for testing pipelines and transformations.
+"""
+
+import shutil
+import tempfile
+from contextlib import contextmanager
+from typing import Any, Dict, Generator, Optional
+
+import numpy as np
+import pandas as pd
+
+
+@contextmanager
+def temp_directory() -> Generator[str, None, None]:
+    """
+    Create a temporary directory for test artifacts.
+
+    Yields:
+        Path to the temporary directory.
+
+    Example:
+        with temp_directory() as temp_dir:
+            path = os.path.join(temp_dir, "test.csv")
+            df.to_csv(path)
+    """
+    temp_dir = tempfile.mkdtemp()
+    try:
+        yield temp_dir
+    finally:
+        shutil.rmtree(temp_dir)
+
+
+def generate_sample_data(
+    rows: int = 10, engine_type: str = "pandas", schema: Optional[Dict[str, str]] = None
+) -> Any:
+    """
+    Generate a sample DataFrame (Pandas or Spark).
+
+    Args:
+        rows: Number of rows to generate
+        engine_type: "pandas" or "spark"
+        schema: Optional dictionary of {column_name: type}
+               Supported types: "int", "float", "str", "date"
+
+    Returns:
+        DataFrame (pd.DataFrame or pyspark.sql.DataFrame)
+    """
+    from datetime import datetime, timedelta
+
+    # Default schema if none provided
+    if not schema:
+        schema = {"id": "int", "value": "float", "category": "str", "timestamp": "date"}
+
+    data = {}
+    for col, dtype in schema.items():
+        if dtype == "int":
+            data[col] = np.random.randint(0, 1000, rows)
+        elif dtype == "float":
+            data[col] = np.random.rand(rows) * 100
+        elif dtype == "str":
+            data[col] = [f"val_{i}" for i in range(rows)]
+        elif dtype == "date":
+            base_date = datetime.now()
+            data[col] = [base_date - timedelta(days=i) for i in range(rows)]
+
+    pdf = pd.DataFrame(data)
+
+    if engine_type == "pandas":
+        return pdf
+
+    if engine_type == "spark":
+        try:
+            from pyspark.sql import SparkSession
+
+            # Try to get existing session or create new one
+            spark = SparkSession.builder.master("local[*]").appName("odibi-test").getOrCreate()
+            return spark.createDataFrame(pdf)
+        except ImportError:
+            raise ImportError("Spark not installed. Run 'pip install odibi[spark]'")
+
+    raise ValueError(f"Unknown engine type: {engine_type}")

odibi/testing/source_pool.py

@@ -0,0 +1,277 @@
+"""
+SourcePool: Deterministic, frozen test data sources for Odibi testing.
+
+Phase 7.B.1 - Preparation only, NO runtime logic.
+
+This module defines the schema and metadata structures for deterministic,
+replayable data sources that exercise all supported Odibi data types and
+ingestion paths.
+"""
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+# ============================================
+# Enums for SourcePool Configuration
+# ============================================
+
+
+class FileFormat(str, Enum):
+    """Supported file formats for source pools."""
+
+    CSV = "csv"
+    JSON = "json"
+    PARQUET = "parquet"
+    AVRO = "avro"
+    DELTA = "delta"
+
+
+class SourceType(str, Enum):
+    """Supported source types for ingestion testing."""
+
+    LOCAL = "local"
+    ADLS_EMULATED = "adls_emulated"
+    AZURE_BLOB_EMULATED = "azure_blob_emulated"
+    SQL_JDBC_LOCAL = "sql_jdbc_local"
+    CLOUDFILES = "cloudfiles"
+
+
+class DataQuality(str, Enum):
+    """Classification of data cleanliness."""
+
+    CLEAN = "clean"  # No nulls, no duplicates, valid types
+    MESSY = "messy"  # Contains nulls, edge cases, type issues
+    MIXED = "mixed"  # Combination of clean and messy partitions
+
+
+class PoolStatus(str, Enum):
+    """Lifecycle status of a source pool."""
+
+    DRAFT = "draft"  # Schema defined, data not yet prepared
+    FROZEN = "frozen"  # Data prepared and hash-verified
+    DEPRECATED = "deprecated"  # Marked for removal
+
+
+# ============================================
+# Schema Definitions (explicit, no inference)
+# ============================================
+
+
+class ColumnSchema(BaseModel):
+    """Explicit column definition - NO runtime inference."""
+
+    name: str = Field(description="Column name")
+    dtype: str = Field(description="Data type (string, int64, float64, bool, datetime, etc.)")
+    nullable: bool = Field(default=False, description="Whether column allows nulls")
+    primary_key: bool = Field(default=False, description="Part of primary key")
+    description: Optional[str] = Field(default=None, description="Column documentation")
+    sample_values: Optional[List[Any]] = Field(
+        default=None,
+        description="Example values for documentation (not used at runtime)",
+    )
+
+
+class TableSchema(BaseModel):
+    """Complete table schema definition."""
+
+    columns: List[ColumnSchema] = Field(description="Ordered list of columns")
+    primary_keys: Optional[List[str]] = Field(
+        default=None, description="List of primary key column names"
+    )
+    partition_columns: Optional[List[str]] = Field(
+        default=None, description="Partition columns (for Delta/Parquet)"
+    )
+
+    @model_validator(mode="after")
+    def validate_pk_columns_exist(self):
+        """Ensure all primary key columns exist in schema."""
+        if self.primary_keys:
+            col_names = {c.name for c in self.columns}
+            for pk in self.primary_keys:
+                if pk not in col_names:
+                    raise ValueError(f"Primary key column '{pk}' not in schema")
+        return self
+
+
+# ============================================
+# Data Characteristics Metadata
+# ============================================
+
+
+class DataCharacteristics(BaseModel):
+    """Metadata about data characteristics for test coverage."""
+
+    row_count: int = Field(ge=0, description="Exact row count (deterministic)")
+    has_nulls: bool = Field(default=False, description="Contains null values")
+    has_duplicates: bool = Field(default=False, description="Contains duplicate keys")
+    has_unicode: bool = Field(default=False, description="Contains non-ASCII characters")
+    has_special_chars: bool = Field(default=False, description="Contains newlines, quotes, etc.")
+    has_empty_strings: bool = Field(default=False, description="Contains empty string values")
+    has_whitespace_issues: bool = Field(
+        default=False, description="Leading/trailing whitespace in strings"
+    )
+    has_type_coercion_cases: bool = Field(
+        default=False, description="Values that may coerce unexpectedly"
+    )
+    date_range: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="Date range {min: ISO date, max: ISO date}",
+    )
+    numeric_ranges: Optional[Dict[str, Dict[str, float]]] = Field(
+        default=None,
+        description="Numeric column ranges {column: {min: v, max: v}}",
+    )
+
+
+# ============================================
+# Integrity & Hashing
+# ============================================
+
+
+class IntegrityManifest(BaseModel):
+    """Cryptographic integrity manifest for frozen source pools."""
+
+    algorithm: Literal["sha256"] = "sha256"
+    file_hashes: Dict[str, str] = Field(description="Map of relative file path -> SHA256 hash")
+    manifest_hash: str = Field(
+        description="SHA256 hash of sorted file_hashes for quick verification"
+    )
+    frozen_at: datetime = Field(description="Timestamp when pool was frozen")
+    frozen_by: str = Field(default="system", description="User/system that froze the pool")
+
+
+# ============================================
+# Source Pool Definition (Main Schema)
+# ============================================
+
+
+class SourcePoolConfig(BaseModel):
+    """
+    Complete SourcePool definition.
+
+    This is the primary schema for defining deterministic test data sources.
+
+    Invariants:
+    - All data is disk-backed and hashable
+    - Schemas are explicit (no runtime inference)
+    - Metadata is complete and machine-readable
+    - Sources are immutable once frozen
+    """
+
+    # === Identification ===
+    pool_id: str = Field(
+        description="Unique identifier (e.g., 'nyc_taxi_csv_clean')",
+        pattern=r"^[a-z][a-z0-9_]*$",
+    )
+    version: str = Field(
+        default="1.0.0",
+        description="Semantic version for tracking pool evolution",
+    )
+    name: str = Field(description="Human-readable name")
+    description: str = Field(description="Detailed description of the dataset")
+
+    # === Source Configuration ===
+    file_format: FileFormat = Field(description="File format")
+    source_type: SourceType = Field(description="Source/ingestion type to test")
+    data_quality: DataQuality = Field(description="Clean/messy/mixed classification")
+
+    # === Schema (Explicit, No Inference) ===
+    schema: TableSchema = Field(description="Explicit schema definition")
+
+    # === Disk Location (relative to .odibi/source_cache/) ===
+    cache_path: str = Field(
+        description="Relative path under .odibi/source_cache/ (e.g., 'nyc_taxi/csv/clean/')"
+    )
+
+    # === Data Characteristics ===
+    characteristics: DataCharacteristics = Field(
+        description="Metadata about data properties for test coverage"
+    )
+
+    # === Status & Integrity ===
+    status: PoolStatus = Field(
+        default=PoolStatus.DRAFT,
+        description="Current lifecycle status",
+    )
+    integrity: Optional[IntegrityManifest] = Field(
+        default=None,
+        description="Integrity manifest (required when status=frozen)",
+    )
+
+    # === Provenance ===
+    original_source: Optional[str] = Field(
+        default=None,
+        description="URL or reference to original public dataset",
+    )
+    license: Optional[str] = Field(
+        default=None,
+        description="Data license (e.g., 'CC0', 'MIT', 'Public Domain')",
+    )
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When this pool definition was created",
+    )
+    updated_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="Last modification timestamp",
+    )
+
+    # === Test Coverage Hints ===
+    tests_coverage: List[str] = Field(
+        default_factory=list,
+        description="List of test scenarios this pool covers (e.g., 'null_handling', 'unicode_support')",
+    )
+    compatible_pipelines: List[str] = Field(
+        default_factory=list,
+        description="Pipeline patterns this pool is designed for (e.g., 'bronze_ingestion', 'silver_dedup')",
+    )
+
+    @model_validator(mode="after")
+    def validate_frozen_has_integrity(self):
+        """Frozen pools must have integrity manifest."""
+        if self.status == PoolStatus.FROZEN and not self.integrity:
+            raise ValueError(f"Pool '{self.pool_id}': frozen status requires integrity manifest")
+        return self
+
+    @field_validator("cache_path")
+    @classmethod
+    def validate_cache_path(cls, v: str) -> str:
+        """Ensure cache_path is relative and safe."""
+        if v.startswith("/") or v.startswith("\\") or ".." in v:
+            raise ValueError(f"cache_path must be relative without '..': {v}")
+        return v.replace("\\", "/")
+
+
+# ============================================
+# Source Pool Index (Registry)
+# ============================================
+
+
+class SourcePoolIndex(BaseModel):
+    """
+    Index of all registered source pools.
+
+    Stored at: .odibi/source_metadata/pool_index.yaml
+    """
+
+    version: str = Field(default="1.0.0", description="Index schema version")
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    pools: Dict[str, str] = Field(
+        default_factory=dict,
+        description="Map of pool_id -> metadata file path (relative to source_metadata/)",
+    )
+
+    def add_pool(self, pool_id: str, metadata_path: str) -> None:
+        """Register a pool in the index."""
+        self.pools[pool_id] = metadata_path
+        self.updated_at = datetime.now(timezone.utc)
+
+    def remove_pool(self, pool_id: str) -> None:
+        """Remove a pool from the index."""
+        if pool_id in self.pools:
+            del self.pools[pool_id]
+            self.updated_at = datetime.now(timezone.utc)