sandwich 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

sandwich/modeling/__init__.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import Table
+
+@dataclass(frozen=True)
+class StgInfo:
+    stg_name: str
+    stg_schema: str
+    hk_keys: dict[str, Any]
+    bk_keys: dict[str, Any]
+    sys_columns: dict[str, Any]
+    bus_columns: dict[str, Any]
+
+class Dv2ModelingMetadata:
+    HEADER_TEMPLATE = """/* 
+    =====================================================================
+    AUTO-GENERATED CODE — DO NOT EDIT MANUALLY
+    =====================================================================
+
+    This stored procedure was automatically generated by:
+      sandwich (https://pypi.org/project/sandwich/)
+
+    Manual changes are discouraged.
+    This file may be regenerated at any time, and all manual edits
+    will be overwritten.
+
+    Created on   : {created_on:%Y-%m-%d %H:%M:%S}
+    Updated on   : {updated_on:%Y-%m-%d %H:%M:%S}
+    Generator    : sandwich v{version}
+    Entity name  : {entity_name}
+
+    =====================================================================
+*/
+"""
+    loaddate = "LoadDate"
+    recordsource = "RecordSource"
+    hashdiff = "HashDiff"
+    is_available = "IsAvailable"
+    names: list[str] = [loaddate, recordsource, hashdiff, is_available]
+    # _dialects_config: dict[str, dict[str, str]] = {
+    #     "mssql": {
+    #         loaddate: "DATETIME2",
+    #         recordsource: "VARCHAR(200)",
+    #         hashdiff: "CHAR(40)",
+    #         is_available: "BIT",
+    #     },
+    #     "postgres": {
+    #         loaddate: "TIMESTAMP",
+    #         recordsource: "TEXT",
+    #         hashdiff: "CHAR(40)",
+    #         is_available: "BOOLEAN",
+    #     },
+    # }
+    def __init__(self):
+        #self.dialect = dialect
+
+        #self.column_types = self._dialects_config[self.dialect]
+        self.required_columns: list[str] = [self.loaddate, self.recordsource]
+
+
+modeling_metadata = Dv2ModelingMetadata()
+
+def get_stg_info(stg: Table) -> StgInfo:
+    hk_keys: dict[str, Any] = {}
+    bk_keys: dict[str, Any] = {}
+    sys_columns: dict[str, Any] = {}
+    bus_columns: dict[str, Any] = {}
+
+    for col in stg.columns.values():
+        if col.name.startswith("hk_"):
+            hk_keys[col.name] = col.type
+        elif col.name.startswith("bk_"):
+            bk_keys[col.name] = col.type
+        elif col.name.startswith("sg_"):
+            raise Exception(f"sg column '{col.name}' is not implemented yet")
+        elif col.name.startswith("ts_"):
+            raise Exception(f"ts column '{col.name}' is not implemented yet")
+        elif col.name in modeling_metadata.names:
+            # type_name = metadata.column_types[col.name]
+            # if not str(col.type).startswith(type_name):
+            #     raise Exception(f"{col.name} column must be `{type_name}` type, but it is `{str(col.type)}`")
+            sys_columns[col.name] = col.type
+        else:
+            bus_columns[col.name] = col.type
+
+    return StgInfo(
+        stg_name=stg.name,
+        stg_schema=stg.schema,
+        hk_keys=hk_keys,
+        bk_keys=bk_keys,
+        sys_columns=sys_columns,
+        bus_columns=bus_columns,
+    )
+
+def infer_template(stg_info: StgInfo):
+    hk_count = len(stg_info.hk_keys)
+    if hk_count == 0:
+        raise Exception("hk column is required for `scd2dim` validation")
+    elif hk_count > 1:
+        return "link2fact"
+    else:
+        return "scd2dim"

sandwich/strategies/__init__.py

@@ -0,0 +1,15 @@
+from .base import SchemaGenerator, Validator
+from .factory import StrategyFactory
+from .link2fact import Link2FactValidator, Link2FactSchemaGenerator
+from .scd2dim import Scd2DimValidator, Scd2DimSchemaGenerator
+
+__all__ = [
+    "Validator",
+    "SchemaGenerator",
+    "StrategyFactory",
+    "Scd2DimValidator",
+    "Scd2DimSchemaGenerator",
+    "Link2FactValidator",
+    "Link2FactSchemaGenerator",
+]
+

sandwich/strategies/base.py

@@ -0,0 +1,44 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Tuple
+
+from sqlalchemy import Table
+
+from src.sandwich.modeling import StgInfo
+
+class ValidationResult:
+    def __init__(self, stg_schema: str, entity_name: str
+                 , bk_keys: list[Tuple[str, Any]]
+                 , hk_key: Tuple[str, Any]
+                 , business_column_types: dict[str, Any]
+                 , system_column_types: dict[str, Any]):
+        self.stg_schema = stg_schema
+        self.entity_name = entity_name
+        self.bk_keys = bk_keys
+        self.hk_key = hk_key
+        self.business_column_types = business_column_types
+        self.system_column_types = system_column_types
+
+class Validator(ABC):
+    @abstractmethod
+    def validate_staging(self, stg_info: StgInfo, verbose: bool = False) -> ValidationResult:
+        pass
+
+
+class SchemaGenerator(ABC):
+    @property
+    @abstractmethod
+    def entity_info(self) -> ValidationResult: ...
+
+    @abstractmethod
+    def make_tables(self) -> dict[str, Table]:
+        pass
+
+    @abstractmethod
+    def make_procedures(
+        self,
+        tables: dict[str, Table],
+        entity_registration_date: datetime = datetime.now()
+    ) -> dict[str, Tuple[str, str, str]]:
+        pass
+

sandwich/strategies/factory.py

@@ -0,0 +1,38 @@
+from src.sandwich.dialects import DialectHandler
+
+from .base import Validator, SchemaGenerator, ValidationResult
+from .link2fact import Link2FactValidator, Link2FactSchemaGenerator
+from .scd2dim import Scd2DimValidator, Scd2DimSchemaGenerator
+
+
+class StrategyFactory:
+    _strategies = {
+        "scd2dim": (Scd2DimValidator, Scd2DimSchemaGenerator),
+        "link2fact": (Link2FactValidator, Link2FactSchemaGenerator),
+    }
+
+    @classmethod
+    def register_strategy(cls, template_name: str, validator_class, generator_class):
+        cls._strategies[template_name] = (validator_class, generator_class)
+
+    @classmethod
+    def create_validator(cls, template: str) -> Validator:
+        if template not in cls._strategies:
+            available = ", ".join(cls._strategies.keys())
+            raise ValueError(f"Unknown template '{template}'. Available templates: {available}")
+
+        validator_class, _ = cls._strategies[template]
+        return validator_class()
+
+    @classmethod
+    def create_generator(cls, template: str, dialect_handler: DialectHandler, validation_result: ValidationResult) -> SchemaGenerator:
+        if template not in cls._strategies:
+            available = ", ".join(cls._strategies.keys())
+            raise ValueError(f"Unknown template '{template}'. Available templates: {available}")
+
+        _, generator_class = cls._strategies[template]
+        return generator_class(dialect_handler, validation_result)
+
+    @classmethod
+    def get_available_templates(cls) -> list[str]:
+        return list(cls._strategies.keys())

sandwich/strategies/link2fact.py

@@ -0,0 +1,91 @@
+"""Link to Fact strategy implementations."""
+from datetime import datetime
+from typing import Tuple
+
+from sqlalchemy import Table
+
+from src.sandwich.dialects.base import DialectHandler
+
+from .base import Validator, SchemaGenerator, ValidationResult
+
+
+class Link2FactValidator(Validator):
+
+    def validate_staging(self, stg_info: StgInfo, verbose: bool = False) -> dict:
+        """Validate staging table for link2fact mode."""
+        if verbose:
+            raise Exception("verbose is not implemented yet")
+
+        # TODO: Implement link2fact specific validation logic
+        # This will likely be different from scd2dim validation
+        # For example: checking for link keys, fact columns, etc.
+
+        bk_keys = []
+        hk_key = None
+        business_column_types = {}
+        system_column_types = {}
+        link_keys = []  # New concept for link2fact
+        fact_columns = []  # New concept for link2fact
+
+        # Placeholder validation logic
+        for col in stg_table.columns.values():
+            # TODO: Implement column classification for link2fact mode
+            pass
+
+        return {
+            "stg_schema": stg_table.schema,
+            "entity_name": stg_table.name,
+            "bk_keys": bk_keys,
+            "hk_key": hk_key,
+            "business_column_types": business_column_types,
+            "system_column_types": system_column_types,
+            "link_keys": link_keys,
+            "fact_columns": fact_columns,
+        }
+
+
+class Link2FactSchemaGenerator(SchemaGenerator):
+
+    def __init__(self, dialect_handler: DialectHandler, validation_result: ValidationResult):
+        self.dialect_handler = dialect_handler
+        self._validation_result = validation_result
+
+    @property
+    def entity_info(self) -> ValidationResult:
+        return self._validation_result
+
+    def make_tables(self) -> dict[str, Table]:
+        """Create link and fact tables for link2fact mode."""
+        # TODO: Implement link2fact table creation
+        # This will create different table structures than scd2dim
+        # For example: link table, fact table (instead of hub/sat/dim)
+
+        entity_name = self._validation_result.entity_name
+
+        # Placeholder - actual implementation needed
+        link_table: Table | None = None
+        fact_table: Table | None = None
+
+        return {
+            "link": link_table,
+            "fact": fact_table,
+        }
+
+    def make_procedures(
+        self,
+        tables: dict[str, Table],
+        entity_registration_date: datetime,
+    ) -> dict[str, Tuple[str, str]]:
+        """Generate procedures for link2fact mode."""
+        procedures = {}
+
+        # TODO: Implement link2fact procedure generation using dialect_handler
+        # This will generate different procedures than scd2dim
+        # For example: link population, fact population, aggregation logic, etc.
+
+        # When implementing, use self.dialect_handler methods to generate SQL
+        # Example:
+        # link_proc_code, link_proc_name = self.dialect_handler.make_link_proc(...)
+        # procedures["link"] = (link_proc_code, link_proc_name)
+
+        return procedures

sandwich/strategies/scd2dim.py

@@ -0,0 +1,246 @@
+from datetime import datetime
+from typing import Iterator, Tuple
+
+from sqlalchemy import Column, MetaData, Table, UniqueConstraint
+
+from src.sandwich import SANDWICH_VERSION
+from src.sandwich.dialects.base import DialectHandler
+from src.sandwich.modeling import modeling_metadata, StgInfo
+
+from .base import Validator, SchemaGenerator, ValidationResult
+
+
+class Scd2DimValidator(Validator):
+
+    def validate_staging(self, stg_info: StgInfo, verbose: bool = False) -> ValidationResult:
+        """Validate staging table or view for `scd2dim` template.
+
+        Raises: Exception"""
+        if verbose:
+            raise Exception("verbose is not implemented yet")
+
+        # -----------------
+        # hk
+        # -----------------
+        # only one hash key is allowed for `scd2dim` profile
+        # and its name should match `hk_[entity_name]` pattern
+        hk_count = len(stg_info.hk_keys)
+        if hk_count == 0:
+            raise Exception("hk column is required for `scd2dim` validation")
+        elif hk_count > 1:
+            raise Exception(f"More than one hk column found in stg.{stg_info.stg_name}")
+        # hk_key = (key_name, key_type)
+        hk_key = list(stg_info.hk_keys.items())[0]
+        if hk_key[0] != f"hk_{stg_info.stg_name}":
+            raise Exception(f"hk column has invalid name '{hk_key[0]}'")
+
+        # -----------------
+        # BKs
+        # -----------------
+        # You don't need a hub or/and a dim tables for a non-business entity.
+        # So you have to have at least one business key, and you can have more.
+        # Naming convention is to just add a `bk_` prefix to the original key name
+        # because we want to keep information of the original names
+        if len(stg_info.bk_keys) == 0:
+            raise Exception("bk column(s) are required for `scd2dim` validation")
+
+
+        system_column_names = stg_info.sys_columns.keys()
+
+        # universal check - all dv2 raw objects should be auditable
+        for required_col in modeling_metadata.required_columns:
+            if required_col not in system_column_names:
+                raise Exception(f"{required_col} column is required")
+
+        # scd2dim specific validations
+        if modeling_metadata.hashdiff not in system_column_names:
+            raise Exception(f"{modeling_metadata.hashdiff} column is required for scd2dim validation")
+        if modeling_metadata.is_available not in system_column_names:
+            raise Exception(f"{modeling_metadata.is_available} column is required for scd2dim validation")
+
+        return ValidationResult(
+            stg_schema=stg_info.stg_schema,
+            entity_name=stg_info.stg_name,
+            bk_keys=[(nm, tp) for nm, tp in stg_info.bk_keys.items()],
+            hk_key=hk_key,
+            business_column_types=stg_info.bus_columns,
+            system_column_types=stg_info.sys_columns,
+        )
+
+class Scd2DimSchemaGenerator(SchemaGenerator):
+
+    def __init__(self, dialect_handler: DialectHandler, validation_result: ValidationResult):
+        self.dialect_handler = dialect_handler
+        self._validation_result = validation_result
+
+    @property
+    def entity_info(self) -> ValidationResult:
+        return self._validation_result
+
+    def make_tables(self) -> dict[str, Table]:
+        entity_name = self._validation_result.entity_name
+        bk_keys = self._validation_result.bk_keys
+        hk_key = self._validation_result.hk_key
+        business_column_types = self._validation_result.business_column_types
+        system_column_types = self._validation_result.system_column_types
+
+        # Helper functions for creating columns
+        def get_bk_columns() -> Iterator[Column]:
+            return (Column(bk_key[0], bk_key[1], nullable=False) for bk_key in bk_keys)
+
+        def get_bk_pk_columns() -> Iterator[Column]:
+            return (Column(bk_key[0], bk_key[1], primary_key=True) for bk_key in bk_keys)
+
+        def get_hk_pk_column() -> Column:
+            return Column(hk_key[0], hk_key[1], primary_key=True)
+
+        def get_loaddate_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column(_load_date, _load_date_type, nullable=False)
+
+        def get_loaddate_pk_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column(_load_date, _load_date_type, primary_key=True)
+
+        def get_datefrom_pk_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column("DateFrom", _load_date_type, primary_key=True)
+
+        def get_dateto_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column("DateTo", _load_date_type, nullable=True)
+
+        def get_recordsource_column() -> Column:
+            _record_source = modeling_metadata.recordsource
+            _record_source_type = system_column_types[_record_source]
+            return Column(_record_source, _record_source_type, nullable=False)
+
+        def get_business_columns() -> Iterator[Column]:
+            return (Column(col_name, col_type, nullable=True) for (col_name, col_type) in business_column_types.items())
+
+        def get_is_available_column() -> Column:
+            _is_available = modeling_metadata.is_available
+            _is_available_type = system_column_types[_is_available]
+            return Column(_is_available, _is_available_type, nullable=False)
+
+        def get_hashdiff_column() -> Column:
+            _hashdiff = modeling_metadata.hashdiff
+            _hashdiff_type = system_column_types[_hashdiff]
+            return Column(_hashdiff, _hashdiff_type, nullable=False)
+
+        # Create hub table
+        hub_table = Table(entity_name, MetaData(), schema="hub")
+        for bk_col in get_bk_columns():
+            hub_table.append_column(bk_col)
+        hub_table.append_column(get_hk_pk_column())
+        hub_table.append_column(get_loaddate_column())
+        hub_table.append_column(get_recordsource_column())
+        hub_table.append_constraint(UniqueConstraint(*[bk[0] for bk in bk_keys]))
+
+        # Create sat table
+        sat_table = Table(entity_name, MetaData(), schema="sat")
+        for bk_col in get_bk_columns():
+            sat_table.append_column(bk_col)
+        sat_table.append_column(get_hk_pk_column())
+        sat_table.append_column(get_loaddate_pk_column())
+        sat_table.append_column(get_recordsource_column())
+        sat_table.append_column(get_hashdiff_column())
+        for business_col in get_business_columns():
+            sat_table.append_column(business_col)
+        sat_table.append_column(get_is_available_column())
+
+        # Create dim table
+        dim_table = Table(entity_name, MetaData(), schema="dim")
+        for bk_col in get_bk_pk_columns():
+            dim_table.append_column(bk_col)
+        for business_col in get_business_columns():
+            dim_table.append_column(business_col)
+        dim_table.append_column(get_is_available_column())
+        dim_table.append_column(Column("IsCurrent", self.dialect_handler.get_boolean_type(), nullable=False))
+        dim_table.append_column(get_datefrom_pk_column())
+        dim_table.append_column(get_dateto_column())
+
+        return {
+            "hub": hub_table,
+            "sat": sat_table,
+            "dim": dim_table,
+        }
+
+    def make_procedures(self, tables: dict[str, Table]
+                        , entity_registration_date: datetime = datetime.now()) -> dict[str, Tuple[str, str, str]]:
+        procedures = {}
+
+        header = modeling_metadata.HEADER_TEMPLATE.format(
+            created_on=entity_registration_date,
+            updated_on=datetime.now(),
+            version=SANDWICH_VERSION,
+            entity_name=self._validation_result.entity_name
+        )
+
+        stg_proc_name = None
+        if self._validation_result.stg_schema == "proxy":
+            stg_proc_code, stg_proc_name, stg_call_stmt = self.dialect_handler.make_stg_materialization_proc(
+                entity_name=self._validation_result.entity_name,
+                header=header
+            )
+            procedures["stg"] = (stg_proc_code, stg_proc_name, stg_call_stmt)
+
+        hub_table = tables["hub"]
+        hub_proc_code, hub_proc_name, hub_call_stmt = self.dialect_handler.make_hub_proc(
+            hub_table=hub_table,
+            bk_keys=self._validation_result.bk_keys,
+            header=header
+        )
+        procedures["hub"] = (hub_proc_code, hub_proc_name, hub_call_stmt)
+
+        # Generate sat procedure
+        sat_table = tables["sat"]
+        sat_proc_code, sat_proc_name, sat_call_stmt = self.dialect_handler.make_sat_proc(
+            sat_table=sat_table,
+            hk_name=self._validation_result.hk_key[0],
+            hashdiff_col=modeling_metadata.hashdiff,
+            is_available_col=modeling_metadata.is_available,
+            loaddate_col=modeling_metadata.loaddate,
+            stg_schema=self._validation_result.stg_schema,
+            header=header
+        )
+        procedures["sat"] = (sat_proc_code, sat_proc_name, sat_call_stmt)
+
+        # Generate dim procedure
+        dim_table = tables["dim"]
+        dim_proc_code, dim_proc_name, dim_call_stmt = self.dialect_handler.make_dim_scd2_proc(
+            dim_table=dim_table,
+            bk_keys=self._validation_result.bk_keys,
+            header=header
+        )
+        procedures["dim"] = (dim_proc_code, dim_proc_name, dim_call_stmt)
+
+        # Generate job procedure
+        job_proc_code, job_proc_name, job_call_stmt = self.dialect_handler.make_job_proc(
+            entity_name=self._validation_result.entity_name,
+            hub_proc_name=hub_proc_name,
+            sat_proc_name=sat_proc_name,
+            dim_proc_name=dim_proc_name,
+            stg_proc_name=stg_proc_name,
+            header=header
+        )
+        procedures["job"] = (job_proc_code, job_proc_name, job_call_stmt)
+
+        # Generate drop procedure
+        drop_proc_code, drop_proc_name, drop_call_stmt = self.dialect_handler.make_drop_proc(
+            entity_name=self._validation_result.entity_name,
+            stg_schema=self._validation_result.stg_schema,
+            job_proc_name=job_proc_name,
+            stg_proc_name=stg_proc_name,
+            hub_proc_name=hub_proc_name,
+            sat_proc_name=sat_proc_name,
+            dim_proc_name=dim_proc_name,
+            header=header
+        )
+        procedures["drop"] = (drop_proc_code, drop_proc_name, drop_call_stmt)
+
+        return procedures