sibi-flux 2025.12.0__py3-none-any.whl

sibi_flux/datacube/_data_cube.py

@@ -0,0 +1,332 @@
+from __future__ import annotations
+import dask.dataframe as dd
+import pandas as pd
+import sqlalchemy.types as sa_types
+from typing import Any, ClassVar, List, Literal, Optional
+from collections.abc import Mapping, MutableMapping
+from pydantic import BaseModel, Field, ConfigDict, SecretStr
+
+from sibi_flux.df_helper import DfHelper
+from sibi_flux.utils import DataUtils
+from sibi_flux.dask_cluster import dask_is_empty
+from sibi_flux.df_validator import DfValidator
+
+# Define types for clarity
+DataFrameType = dd.DataFrame | pd.DataFrame
+
+
+class DatacubeConfig(BaseModel):
+    """
+    Pydantic model for strict validation of Datacube configuration.
+    Allows extra fields to pass through to DfHelper/Backend.
+    """
+
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
+
+    table: Optional[str] = None
+    backend: str = "sqlalchemy"
+    connection_url: Optional[str | SecretStr] = None
+    field_map: Mapping[str, str] = Field(default_factory=dict)
+    legacy_filters: bool = True
+    debug: bool = False
+
+    # Validation Config
+    validation_schema: Mapping[str, str] = Field(default_factory=dict)
+    enforce_schema: bool = False
+
+    # Allow optional logger instance
+    logger: Optional[Any] = None
+
+
+class Datacube(DfHelper):
+    """
+    Enhanced Datacube that simplifies subclassing via declarative configuration
+    and automated common transformations.
+
+    Consolidates functionality from the legacy BaseDatacube.
+
+    Attributes:
+        table (str): Table name for database connection.
+        backend (str): Backend type (e.g., 'sqlalchemy').
+        connection_url (str): Database connection URL.
+        field_map (Mapping): Mapping of field names.
+        legacy_filters (bool): Use legacy filters.
+        live (bool): Live data flag.
+
+        boolean_columns (List[str]): Columns to convert to boolean.
+        numeric_float_columns (List[str]): Columns to convert to float.
+        numeric_int_columns (List[str]): Columns to convert to int.
+        date_fields (List[str]): Columns to convert to datetime.
+    """
+
+    # Declarative Config Attributes
+    table: str = ""
+    backend: Literal["sqlalchemy", "parquet", "http"] = "sqlalchemy"
+    connection_url: str = ""
+    field_map: Mapping[str, str] = {}
+    legacy_filters: bool = True
+    pii_columns: List[str] = []
+
+    # Validation
+    validation_schema: ClassVar[Mapping[str, str]] = {}
+    enforce_schema: ClassVar[bool] = False
+
+    # Declarative Transformation Lists
+    boolean_columns: List[str] = []
+    numeric_float_columns: List[str] = []
+    numeric_int_columns: List[str] = []
+    date_fields: List[str] = []
+
+    # Config container compatible with DfHelper
+    config: MutableMapping[str, Any] = {}
+
+    def __init__(self, **kwargs):
+        # State container
+        self.df: Optional[DataFrameType] = None
+        self._schema_inferred = False
+
+        # Build config dictionary from class attributes if not already present
+        if not self.config:
+            self.config = {
+                "table": self.table,
+                "backend": self.backend,
+                "connection_url": self.connection_url,
+                "field_map": self.field_map,
+                "legacy_filters": self.legacy_filters,
+                "validation_schema": self.validation_schema,
+                "enforce_schema": self.enforce_schema,
+            }
+
+        # Merge kwargs into the preliminary config
+        raw_config = {**self.config, **kwargs}
+
+        # Validate configuration using Pydantic
+        # This ensures types are correct and provides clear checks
+        validated_model = DatacubeConfig.model_validate(raw_config)
+
+        # Update self.config with the validated, normalized dictionary
+        self.config = validated_model.model_dump(exclude_unset=True)
+
+        # Copy class-level lists to instance to avoid mutation of shared class state
+        # and to allow instance-specific inference
+        self.boolean_columns = list(self.boolean_columns)
+        self.numeric_float_columns = list(self.numeric_float_columns)
+        self.numeric_int_columns = list(self.numeric_int_columns)
+        self.date_fields = list(self.date_fields)
+
+        # Initialize DfHelper with validated config
+        # We model_dump() allowing extra fields so that backend-specific params
+        # (like pool_size) are preserved and passed to DfHelper.
+        super().__init__(**self.config)
+
+        # Store dask_client if provided, for resilience checks and external access
+        self.dask_client = self.config.get("dask_client")
+
+    def _infer_schema(self):
+        """
+        Attempts to infer column types from the SQLAlchemy model reflection.
+        Populates the transformation lists if they are missing metadata.
+        """
+        if self.config.get("backend") != "sqlalchemy":
+            return
+
+        try:
+            # This triggers reflection matching the table name
+            conn = self.get_sql_connection()
+            model = conn.model
+            if not model:
+                return
+
+            field_map = self.config.get("field_map", {})
+
+            for col in model.__table__.columns:
+                # Map DB column name to DataFrame column name (if renamed via field_map)
+                df_col_name = field_map.get(col.name, col.name)
+
+                # Check types and append if not already present
+                if isinstance(col.type, sa_types.Boolean):
+                    if df_col_name not in self.boolean_columns:
+                        self.boolean_columns.append(df_col_name)
+
+                elif isinstance(
+                    col.type,
+                    (sa_types.Integer, sa_types.BigInteger, sa_types.SmallInteger),
+                ):
+                    if (
+                        df_col_name not in self.numeric_int_columns
+                        and df_col_name not in self.boolean_columns
+                        and df_col_name not in self.numeric_float_columns
+                    ):
+                        self.numeric_int_columns.append(df_col_name)
+
+                elif isinstance(col.type, (sa_types.Float, sa_types.Numeric)):
+                    # Numeric/Decimal often treated as float in analysis unless strict
+                    if df_col_name not in self.numeric_float_columns:
+                        self.numeric_float_columns.append(df_col_name)
+
+                elif isinstance(
+                    col.type, (sa_types.Date, sa_types.DateTime, sa_types.TIMESTAMP)
+                ):
+                    if df_col_name not in self.date_fields:
+                        self.date_fields.append(df_col_name)
+
+        except Exception as e:
+            # Inference is an enhancement; failure shouldn't block app start
+            # but we log it for debugging.
+            self.logger.debug(f"Schema inference skipped: {e}")
+
+    def _validate(self, df: DataFrameType) -> DataFrameType:
+        """
+        Runs DfValidator if a schema is configured.
+        """
+        schema = self.config.get("validation_schema")
+        if not schema:
+            return df
+
+        validator = DfValidator(
+            df, logger=self.logger, debug=self.config.get("debug", False)
+        )
+
+        # 1. Standardize (always good practice if validating)
+        # Note: We skip full standardization for now to avoid side effects on legacy names,
+        # but we could adhere to stricter contracts here.
+
+        if self.config.get("enforce_schema", False):
+            validator.apply_schema_map(schema)
+        else:
+            try:
+                validator.validate_schema(schema)
+            except TypeError as e:
+                # Re-raise to block pipeline
+                self.logger.error(f"Schema Validation Failed: {e}")
+                raise e
+
+        return validator.get_df()
+
+    def get_ddl(self, table_name: Optional[str] = None) -> str:
+        """
+        Generates ClickHouse DDL for the current cube.
+        Requires that self.df is loaded (or at least inferred).
+        """
+        if self.df is None:
+            raise RuntimeError("Cannot generate DDL: Datacube has not been loaded.")
+
+        name = table_name or self.config.get("table") or "datacube_table"
+        validator = DfValidator(self.df, logger=self.logger)
+        return validator.generate_clickhouse_ddl(name)
+
+    # -----------------------------------------------------------------------
+    # Lifecycle (Consolidated from BaseDatacube)
+    # -----------------------------------------------------------------------
+    def load(self, **kwargs) -> DataFrameType:
+        """
+        Synchronous load pipeline.
+        """
+        # 1. Load Raw Data (Delegate to DfHelper)
+        df = super().load(**kwargs)
+
+        # 2. Check Emptiness (Resilient)
+        if not self._is_empty(df):
+            # 3. Apply Transform Hook
+            df = self.fix_data(df, **kwargs)
+            # 4. Validate
+            df = self._validate(df)
+        else:
+            self.logger.debug(f"No data loaded by {self.__class__.__name__}")
+
+        # 4. Update State & Return
+        self.df = df
+        return df
+
+    async def aload(self, **kwargs) -> DataFrameType:
+        """
+        Asynchronous load pipeline.
+        """
+        # 1. Load Raw Data
+        df = await super().aload(**kwargs)
+
+        # 2. Check Emptiness (Async Safe offload)
+        import asyncio
+
+        is_empty = await asyncio.to_thread(self._is_empty, df)
+
+        if not is_empty:
+            # 3. Apply Async Transform Hook
+            df = await self.afix_data(df, **kwargs)
+            # 4. Validate (CPU bound)
+            df = await asyncio.to_thread(self._validate, df)
+        else:
+            self.logger.debug(f"No data loaded by {self.__class__.__name__}")
+
+        # 4. Update State & Return
+        self.df = df
+        return df
+
+    # -----------------------------------------------------------------------
+    # Transformation Hooks
+    # -----------------------------------------------------------------------
+    def fix_data(self, df: DataFrameType, **kwargs) -> DataFrameType:
+        """
+        Applies declarative transformations automatically.
+        Infers schema types on first run if using SQLAlchemy backend.
+        Subclasses can override this to add custom logic, calling super().fix_data(df) first.
+        """
+        if not self._schema_inferred:
+            self._infer_schema()
+            self._schema_inferred = True
+
+        utils = DataUtils(logger=self.logger)
+
+        if self.pii_columns:
+            df = utils.transform_pii_columns(df, self.pii_columns)
+
+        if self.boolean_columns:
+            df = utils.transform_boolean_columns(df, self.boolean_columns)
+
+        if self.numeric_float_columns:
+            df = utils.transform_numeric_columns(
+                df, columns=self.numeric_float_columns, dtype=float
+            )
+
+        if self.numeric_int_columns:
+            df = utils.transform_numeric_columns(
+                df, columns=self.numeric_int_columns, dtype=int
+            )
+
+        if self.date_fields:
+            if isinstance(df, dd.DataFrame):
+                df = utils.convert_to_datetime_dask(df, self.date_fields)
+            else:
+                for col in self.date_fields:
+                    if col in df.columns:
+                        df[col] = pd.to_datetime(
+                            df[col], errors="coerce", utc=True, format="mixed"
+                        )
+
+        return df
+
+    async def afix_data(self, df: DataFrameType, **kwargs) -> DataFrameType:
+        """
+        Asynchronous transformation hook.
+        Offloads synchronous fix_data (and schema inference) to a thread.
+        """
+        import asyncio
+
+        return await asyncio.to_thread(self.fix_data, df, **kwargs)
+
+    # -----------------------------------------------------------------------
+    # Internals
+    # -----------------------------------------------------------------------
+    def _is_empty(self, df: Optional[DataFrameType]) -> bool:
+        """
+        Robust emptiness check using dask_cluster.
+        """
+        if df is None:
+            return True
+
+        if isinstance(df, pd.DataFrame):
+            return df.empty
+
+        # Use resilient checker
+        client = getattr(self, "dask_client", None)
+        return dask_is_empty(df, dask_client=client, logger=self.logger)

sibi_flux/datacube/config_engine.py

@@ -0,0 +1,152 @@
+import re
+import yaml
+from pathlib import Path
+from typing import Literal, Optional, Any
+from collections.abc import Mapping, Sequence
+from pydantic import BaseModel
+
+# --- Schema Definitions ---
+
+
+class GlobalSettings(BaseModel):
+    cubes_root_path: str
+    fields_module_root: str
+    default_db_conn: str
+    env_file: Optional[str] = ".env.linux"
+    exclusions: list[str] = []
+
+
+class DiscoveryRule(BaseModel):
+    pattern: str
+    match_type: Literal["prefix", "exact", "regex"] = "prefix"
+    domain: str
+    output_template: str
+
+    # Optional: Override the DB connection for specific tables
+    db_conn_override: Optional[str] = None
+
+
+class GeneratorConfig(BaseModel):
+    version: float
+    settings: GlobalSettings
+    discovery_rules: list[DiscoveryRule]
+
+
+# --- Logic Engine ---
+
+
+class ConfigurationEngine:
+    def __init__(self, config_path: str = "generator_config.yaml"):
+        # Resolve config path relative to this file if not absolute
+        path = Path(config_path)
+        if not path.is_absolute():
+            path = Path(__file__).parent / config_path
+
+        self.config_path = path
+        self.config = self._load_config()
+
+    def _load_config(self) -> GeneratorConfig:
+        if not self.config_path.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config_path}")
+
+        with open(self.config_path, "r") as f:
+            raw_data = yaml.safe_load(f)
+        return GeneratorConfig(**raw_data)
+
+    def resolve_table(self, table_name: str) -> Optional[dict[str, Any]]:
+        """
+        Iterates through rules to find a match.
+        Returns a dict with resolved paths and metadata.
+        """
+        # 1. Check Global Exclusions
+        for pattern in self.config.settings.exclusions:
+            # We assume exclusions are regex patterns for flexibility
+            if re.search(pattern, table_name):
+                return None
+
+        # 2. Check Rules
+        for rule in self.config.discovery_rules:
+            match_data = self._check_match(rule, table_name)
+            if match_data is not None:
+                return self._build_result(rule, table_name, match_data)
+
+        return None  # No match found
+
+    def _check_match(
+        self, rule: DiscoveryRule, table_name: str
+    ) -> Optional[dict[str, str]]:
+        """
+        Checks if rule matches table_name.
+        Returns capturing groups dict if match (empty dict for non-regex matches), None otherwise.
+        """
+        if rule.match_type == "exact":
+            if table_name == rule.pattern:
+                return {}
+
+        elif rule.match_type == "prefix":
+            if table_name.startswith(rule.pattern):
+                return {}
+
+        elif rule.match_type == "regex":
+            m = re.search(rule.pattern, table_name)
+            if m:
+                # Combine numbered groups (as strings) and named groups
+                groups = {str(i + 1): g for i, g in enumerate(m.groups())}
+                groups.update(m.groupdict())
+                return groups
+
+        return None
+
+    def _build_result(
+        self, rule: DiscoveryRule, table_name: str, groups: dict[str, str]
+    ) -> dict[str, Any]:
+        """Constructs the final paths based on global settings + rule + captured groups"""
+
+        # 1. Resolve Output Subpath with Template Substitution
+        # Supports {feature}, {1}, etc.
+        try:
+            subpath = rule.output_template.format(**groups)
+        except KeyError:
+            # Fallback for safe substitution if keys missing? Or let it raise?
+            # Let's try to be safe but warn? For now, strict formatting.
+            subpath = rule.output_template
+
+        # 2. Resolve Physical Path
+        root = Path(self.config.settings.cubes_root_path)
+        full_output_path = str(root / subpath)
+
+        # 3. Resolve Field Map Path (Smart Guessing)
+        # e.g., solutions.conf.transforms.fields.logistics.asm_tracking_X.field_map
+        field_map_module = (
+            f"{self.config.settings.fields_module_root}."
+            f"{rule.domain}.{table_name}.field_map"
+        )
+
+        return {
+            "domain": rule.domain,
+            "path": full_output_path,  # 'path' key is expected by generator
+            "save_to_path": full_output_path,
+            "field_map": field_map_module,  # 'field_map' template logic relies on discovery result often
+            "connection_obj": rule.db_conn_override
+            or self.config.settings.default_db_conn,
+        }
+
+
+# --- Module Level Interface ---
+
+_ENGINE = None
+
+
+def get_engine() -> ConfigurationEngine:
+    global _ENGINE
+    if _ENGINE is None:
+        _ENGINE = ConfigurationEngine()
+    return _ENGINE
+
+
+def match_rule(table_name: str) -> Optional[dict[str, str]]:
+    """
+    Finds the first matching rule for a table name.
+    """
+    engine = get_engine()
+    return engine.resolve_table(table_name)

sibi_flux/datacube/field_factory.py

@@ -0,0 +1,48 @@
+from typing import List, MutableMapping, Optional
+from pathlib import Path
+from .field_registry import GlobalFieldRegistry
+
+# Singleton instance to avoid reloading YAML multiple times
+_SHARED_REGISTRY: Optional[GlobalFieldRegistry] = None
+
+
+class FieldMapFactory:
+    """
+    Factory for creating field translation maps.
+    Uses a shared GlobalFieldRegistry instance to translate field names.
+    """
+
+    @classmethod
+    def create(cls, table_name: str, columns: List[str]) -> MutableMapping[str, str]:
+        """
+        Creates a field map dictionary for the given table and columns.
+
+        Args:
+            table_name: The name of the database table (for context/logging if needed).
+            columns: List of column names to translate.
+
+        Returns:
+            A MutableMapping (dict) where key=column_name, value=translated_name.
+        """
+        registry = cls._get_registry()
+
+        mapping = {}
+        for col in columns:
+            # We use the registry solely for lookups.
+            # If the key isn't there, we default to identity.
+            if col in registry:
+                mapping[col] = registry[col]
+            else:
+                mapping[col] = col
+
+        return mapping
+
+    @classmethod
+    def _get_registry(cls) -> GlobalFieldRegistry:
+        global _SHARED_REGISTRY
+        if _SHARED_REGISTRY is None:
+            # Assume standard location relative to project root
+            registry_path = Path("solutions/conf/global_field_translations.yaml")
+            _SHARED_REGISTRY = GlobalFieldRegistry(registry_path)
+
+        return _SHARED_REGISTRY

sibi_flux/datacube/field_registry.py

@@ -0,0 +1,122 @@
+import yaml
+import threading
+from pathlib import Path
+from typing import Dict, Iterator, Optional, Union
+from collections.abc import MutableMapping
+
+
+class GlobalFieldRegistry(MutableMapping):
+    """
+    Acts as the source of truth for field translations (Database Column -> English Attribute).
+    Backed by a local YAML file. Thread-safe.
+    """
+
+    def __init__(self, file_path: Union[str, Path]):
+        """
+        Initialize the registry.
+
+        Args:
+            file_path: Path to the YAML file backing this registry.
+        """
+        self.file_path = Path(file_path).resolve()
+        self._data: Dict[str, str] = {}
+        self._dirty = False
+        self._lock = threading.RLock()
+        self._load()
+
+    def _load(self) -> None:
+        """Load state from the YAML file."""
+        with self._lock:
+            if self.file_path.exists():
+                try:
+                    with open(self.file_path, "r", encoding="utf-8") as f:
+                        content = yaml.safe_load(f)
+                        if content and isinstance(content, dict):
+                            self._data = content
+                        else:
+                            self._data = {}
+                except Exception:
+                    self._data = {}
+            else:
+                self._data = {}
+
+    def save(self) -> None:
+        """Persist changes to the YAML file if any changes were made."""
+        with self._lock:
+            if not self._dirty:
+                return
+
+            # Ensure directory exists
+            self.file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(self.file_path, "w", encoding="utf-8") as f:
+                for key, value in sorted(self._data.items()):
+                    # Dump single line
+                    # Note: yaml.dump adds a newline at the end, so we strip it
+                    line = yaml.dump(
+                        {key: value},
+                        sort_keys=False,
+                        allow_unicode=True,
+                        default_flow_style=False,
+                    ).strip()
+
+                    # Append comment if translation matches original (and wasn't manually set to something else)
+                    # We rely on the convention that non-translated fields equal their keys.
+                    if key == value:
+                        line += " # Translation missing"
+
+                    f.write(line + "\n")
+
+            self._dirty = False
+
+    def register_field(
+        self,
+        original_name: str,
+        suggested_translation: Optional[str] = None,
+        force_update: bool = False,
+    ) -> None:
+        """
+        Register a field if it does not exist, or update if force_update is True.
+
+        Args:
+            original_name: The original database column name.
+            suggested_translation: Proposed English translation.
+            force_update: If True, overwrite existing translation.
+        """
+        with self._lock:
+            if original_name in self._data:
+                if force_update and suggested_translation:
+                    if self._data[original_name] != suggested_translation:
+                        self._data[original_name] = suggested_translation
+                        self._dirty = True
+            else:
+                # New field - use suggestion or default to identity
+                val = suggested_translation if suggested_translation else original_name
+                self._data[original_name] = val
+                self._dirty = True
+
+    # --- MutableMapping Implementation ---
+
+    def __getitem__(self, key: str) -> str:
+        with self._lock:
+            return self._data[key]
+
+    def __setitem__(self, key: str, value: str) -> None:
+        with self._lock:
+            if self._data.get(key) != value:
+                self._data[key] = value
+                self._dirty = True
+
+    def __delitem__(self, key: str) -> None:
+        with self._lock:
+            if key in self._data:
+                del self._data[key]
+                self._dirty = True
+
+    def __iter__(self) -> Iterator[str]:
+        with self._lock:
+            return iter(list(self._data))
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._data)