sandwich 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

sandwich/modeling/strategies/link2fact.py

@@ -0,0 +1,225 @@
+"""Link to Fact strategy implementations."""
+from datetime import datetime
+from typing import Tuple
+
+from sqlalchemy import Column, MetaData, Table, UniqueConstraint
+
+from sandwich import SANDWICH_VERSION
+from sandwich.dialects.base import DialectHandler
+from sandwich.modeling import modeling_metadata, StgInfo, Dv2SystemInfo
+
+from .base import BaseValidator, SchemaGenerator, ValidationResult
+
+
+class Link2FactValidator(BaseValidator):
+    def __init__(self, template: str):
+        super().__init__(template)
+        self._on_validate_staging = self._validate_staging
+
+    @staticmethod
+    def _validate_staging(stg_info: StgInfo, sys_info: Dv2SystemInfo) -> None:
+        # -----------------
+        # hk
+        # -----------------
+        hk_count = len(stg_info.hk_keys)
+        stg_full_name = f"stg.{stg_info.stg_name}"
+
+        # so only exactly 2 or 3 columns allowed right now
+        if hk_count < 2: # own and at least one foreign
+            raise Exception(f"At least 2 hk columns expected in `{stg_full_name}` for the `link2fact` template")
+        if hk_count > 3:
+            raise Exception(f"{hk_count} hk columns in `{stg_full_name}` for the `link2fact` template?! Are you sure?")
+        if stg_info.degenerate_field is None:
+            raise Exception(f"Degenerate field is required for `{stg_full_name}` for the `link2fact` template")
+
+        # hk_key = (key_name, key_type)
+        hk_keys_copy = list(stg_info.hk_keys.keys()).copy()
+        is_own_hk_found = False
+        expected_own_hk_column_name = f"hk_{stg_info.stg_name}"
+
+        for hk_name in stg_info.hk_keys.keys():
+            if hk_name == expected_own_hk_column_name:
+                hk_keys_copy.remove(hk_name)
+                is_own_hk_found = True
+            else:
+                # check that `name` from `hk_[name]` is existing entity
+                for en in sys_info.entities_list:
+                    if hk_name == f"hk_{en.entity_name}":
+                        hk_keys_copy.remove(hk_name)
+
+        if not is_own_hk_found:
+            raise Exception(f"Column `{expected_own_hk_column_name}` has not been found in `{stg_full_name}`")
+
+        if len(hk_keys_copy) > 0:
+            raise Exception(f"There are no entities related to `{", ".join(hk_keys_copy)}` columns in the `{stg_full_name}`")
+
+        # -----------------
+        # BKs
+        # -----------------
+        # it should be a warning, not an error
+        if len(stg_info.bk_keys) > 0:
+            raise Exception("You dont need bk columns for the `link2fact` template")
+
+
+class Link2FactSchemaGenerator(SchemaGenerator):
+
+    def __init__(self, dialect_handler: DialectHandler, validation_result: ValidationResult):
+        self.dialect_handler = dialect_handler
+        self._validation_result = validation_result
+
+    def make_tables(self) -> dict[str, Table]:
+        return {
+            "link": self.make_link_table(),
+            "sat": self.make_sat_table(),
+            #"fact": self.make_fact_table(),
+        }
+
+    def make_link_table(self) -> Table:
+        entity_name = self._validation_result.entity_name
+
+        # Create link table
+        link_table = Table(entity_name, MetaData(), schema="link")
+        uks: list[str] = []
+
+        # HKs (own and FKs)
+        for hk_key in self._validation_result.hk_keys:
+            if hk_key[0] == f"hk_{entity_name}":
+                col = Column(hk_key[0], hk_key[1], primary_key=True)
+            else:
+                uks.append(hk_key[0])
+                col = Column(hk_key[0], hk_key[1], nullable=False)
+            link_table.append_column(col)
+        dg_key = self._validation_result.degenerate_field
+        link_table.append_column(Column(dg_key[0], dg_key[1], nullable=False))
+        uks.append(dg_key[0])
+        link_table.append_constraint(UniqueConstraint(*uks))
+
+        # LoadDate
+        load_date = modeling_metadata.loaddate
+        load_date_type = self._validation_result.system_column_types[load_date]
+        load_date_col = Column(load_date, load_date_type, nullable=False)
+        link_table.append_column(load_date_col)
+
+        # RecordSource
+        record_source = modeling_metadata.recordsource
+        record_source_type = self._validation_result.system_column_types[record_source]
+        record_source_col = Column(record_source, record_source_type, nullable=False)
+        link_table.append_column(record_source_col)
+
+        return link_table
+
+    def make_sat_table(self) -> Table:
+        entity_name = self._validation_result.entity_name
+
+        # Create sat table
+        sat_table = Table(entity_name, MetaData(), schema="sat")
+
+        # own HK
+        for hk_key in self._validation_result.hk_keys:
+            if hk_key[0] == f"hk_{entity_name}":
+                col = Column(hk_key[0], hk_key[1], primary_key=True)
+                sat_table.append_column(col)
+        # for transactional links
+        dg_key = self._validation_result.degenerate_field
+        if dg_key is not None:
+            sat_table.append_column(Column(dg_key[0], dg_key[1], nullable=False))
+
+
+        # LoadDate
+        load_date = modeling_metadata.loaddate
+        load_date_type = self._validation_result.system_column_types[load_date]
+        load_date_col = Column(load_date, load_date_type, nullable=False)
+        sat_table.append_column(load_date_col)
+
+        # RecordSource
+        record_source = modeling_metadata.recordsource
+        record_source_type = self._validation_result.system_column_types[record_source]
+        record_source_col = Column(record_source, record_source_type, nullable=False)
+        sat_table.append_column(record_source_col)
+
+        for (name_, type_) in self._validation_result.business_column_types.items():
+            col = Column(name_, type_, nullable=True)
+            sat_table.append_column(col)
+
+        return sat_table
+
+    def make_fact_table(self) -> Table:
+        entity_name = self._validation_result.entity_name
+
+        # Create a fact table
+        fact_table = Table(entity_name, MetaData(), schema="fact")
+
+        # not own HKs only
+        for hk_key in self._validation_result.hk_keys:
+            if hk_key[0] != f"hk_{entity_name}":
+                col = Column(hk_key[0], hk_key[1], primary_key=True)
+                fact_table.append_column(col)
+
+        for (name_, type_) in self._validation_result.business_column_types.items():
+            col = Column(name_, type_, nullable=True)
+            fact_table.append_column(col)
+
+        return fact_table
+
+    def make_procedures(self, tables: dict[str, Table]
+                        , entity_registration_date: datetime = datetime.now()) -> dict[str, Tuple[str, str, str]]:
+        procedures = {}
+
+        header = modeling_metadata.HEADER_TEMPLATE.format(
+            created_on=entity_registration_date,
+            updated_on=datetime.now(),
+            version=SANDWICH_VERSION,
+            entity_name=self._validation_result.entity_name
+        )
+
+        stg_proc_name = None
+        if self._validation_result.stg_schema == "proxy":
+            stg_proc_code, stg_proc_name, stg_call_stmt = self.dialect_handler.make_stg_materialization_proc(
+                entity_name=self._validation_result.entity_name,
+                header=header
+            )
+            procedures["stg"] = (stg_proc_code, stg_proc_name, stg_call_stmt)
+
+        link_table = tables["link"]
+        link_proc_code, link_proc_name, link_call_stmt = self.dialect_handler.make_link_proc(
+            link_table=link_table,
+            hk_keys=self._validation_result.hk_keys + [self._validation_result.degenerate_field],
+            header=header
+        )
+        procedures["link"] = (link_proc_code, link_proc_name, link_call_stmt)
+
+        sat_table = tables["sat"]
+        sat_proc_code, sat_proc_name, sat_call_stmt = self.dialect_handler.make_scd0_sat_proc(
+            sat_table=sat_table,
+            header=header
+        )
+        procedures["sat"] = (sat_proc_code, sat_proc_name, sat_call_stmt)
+
+        # job procedure
+        job_proc_names = [] # order-sensitive
+        if self._validation_result.stg_schema == "proxy":
+            job_proc_names.append(stg_proc_name)
+        job_proc_names.extend([link_proc_name, sat_proc_name])
+        job_proc_code, job_proc_name, job_call_stmt = self.dialect_handler.make_job_proc(
+            entity_name=self._validation_result.entity_name,
+            proc_names=job_proc_names,
+            header=header
+        )
+        procedures["job"] = (job_proc_code, job_proc_name, job_call_stmt)
+
+        # drop procedure
+        drop_table_schemas = ["link", "sat"]
+        if self._validation_result.stg_schema == "proxy":
+            drop_table_schemas.append("stg")
+        drop_proc_names = [link_proc_name, sat_proc_name]
+        if self._validation_result.stg_schema == "proxy":
+            drop_proc_names.append(stg_proc_name)
+        drop_proc_code, drop_proc_name, drop_call_stmt = self.dialect_handler.make_drop_proc(
+            entity_name=self._validation_result.entity_name,
+            table_schemas=drop_table_schemas,
+            procedures=drop_proc_names,
+            header=header
+        )
+        procedures["drop"] = (drop_proc_code, drop_proc_name, drop_call_stmt)
+
+        return procedures

sandwich/modeling/strategies/scd2dim.py

@@ -0,0 +1,228 @@
+from datetime import datetime
+from typing import Iterator, Tuple
+
+from sqlalchemy import Column, MetaData, Table, UniqueConstraint
+
+from sandwich import SANDWICH_VERSION
+from sandwich.dialects.base import DialectHandler
+from sandwich.modeling import modeling_metadata, StgInfo, Dv2SystemInfo
+
+from .base import BaseValidator, SchemaGenerator, ValidationResult
+
+
+class Scd2DimValidator(BaseValidator):
+    def __init__(self, template: str):
+        super().__init__(template)
+        self._on_validate_staging = self._validate_staging
+
+    @staticmethod
+    def _validate_staging(stg_info: StgInfo, _: Dv2SystemInfo) -> None:
+        # -----------------
+        # hk
+        # -----------------
+        # only one hash key is allowed for `scd2dim` profile
+        # and its name should match `hk_[entity_name]` pattern
+        hk_count = len(stg_info.hk_keys)
+        if hk_count == 0:
+            raise Exception("hk column is required for `scd2dim` validation")
+        elif hk_count > 1:
+            raise Exception(f"More than one hk column found in stg.{stg_info.stg_name}")
+        # hk_key = (key_name, key_type)
+        hk_key = list(stg_info.hk_keys.items())[0]
+        if hk_key[0] != f"hk_{stg_info.stg_name}":
+            raise Exception(f"hk column has invalid name '{hk_key[0]}'")
+
+        # -----------------
+        # BKs
+        # -----------------
+        # You don't need a hub or/and a dim tables for a non-business entity.
+        # So you have to have at least one business key, and you can have more.
+        # Naming convention is to just add a `bk_` prefix to the original key name
+        # because we want to keep information of the original names
+        if len(stg_info.bk_keys) == 0:
+            raise Exception("bk column(s) are required for `scd2dim` validation")
+
+
+        system_column_names = stg_info.sys_columns.keys()
+
+        if modeling_metadata.hashdiff not in system_column_names:
+            raise Exception(f"{modeling_metadata.hashdiff} column is required for scd2dim validation")
+        if modeling_metadata.is_available not in system_column_names:
+            raise Exception(f"{modeling_metadata.is_available} column is required for scd2dim validation")
+
+class Scd2DimSchemaGenerator(SchemaGenerator):
+
+    def __init__(self, dialect_handler: DialectHandler, validation_result: ValidationResult):
+        self.dialect_handler = dialect_handler
+        self._validation_result = validation_result
+
+    def make_tables(self) -> dict[str, Table]:
+        entity_name = self._validation_result.entity_name
+        bk_keys = self._validation_result.bk_keys
+        hk_key = self._validation_result.hk_keys[0]
+        business_column_types = self._validation_result.business_column_types
+        system_column_types = self._validation_result.system_column_types
+
+        # Helper functions for creating columns
+        def get_bk_columns() -> Iterator[Column]:
+            return (Column(bk_key[0], bk_key[1], nullable=False) for bk_key in bk_keys)
+
+        def get_bk_pk_columns() -> Iterator[Column]:
+            return (Column(bk_key[0], bk_key[1], primary_key=True) for bk_key in bk_keys)
+
+        def get_hk_pk_column() -> Column:
+            return Column(hk_key[0], hk_key[1], primary_key=True)
+
+        def get_loaddate_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column(_load_date, _load_date_type, nullable=False)
+
+        def get_loaddate_pk_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column(_load_date, _load_date_type, primary_key=True)
+
+        def get_datefrom_pk_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column("DateFrom", _load_date_type, primary_key=True)
+
+        def get_dateto_column() -> Column:
+            _load_date = modeling_metadata.loaddate
+            _load_date_type = system_column_types[_load_date]
+            return Column("DateTo", _load_date_type, nullable=True)
+
+        def get_recordsource_column() -> Column:
+            _record_source = modeling_metadata.recordsource
+            _record_source_type = system_column_types[_record_source]
+            return Column(_record_source, _record_source_type, nullable=False)
+
+        def get_business_columns() -> Iterator[Column]:
+            return (Column(col_name, col_type, nullable=True) for (col_name, col_type) in business_column_types.items())
+
+        def get_is_available_column() -> Column:
+            _is_available = modeling_metadata.is_available
+            _is_available_type = system_column_types[_is_available]
+            return Column(_is_available, _is_available_type, nullable=False)
+
+        def get_hashdiff_column() -> Column:
+            _hashdiff = modeling_metadata.hashdiff
+            _hashdiff_type = system_column_types[_hashdiff]
+            return Column(_hashdiff, _hashdiff_type, nullable=False)
+
+        # Create hub table
+        hub_table = Table(entity_name, MetaData(), schema="hub")
+        for bk_col in get_bk_columns():
+            hub_table.append_column(bk_col)
+        hub_table.append_column(get_hk_pk_column())
+        hub_table.append_column(get_loaddate_column())
+        hub_table.append_column(get_recordsource_column())
+        hub_table.append_constraint(UniqueConstraint(*[bk[0] for bk in bk_keys]))
+
+        # Create sat table
+        sat_table = Table(entity_name, MetaData(), schema="sat")
+        for bk_col in get_bk_columns():
+            sat_table.append_column(bk_col)
+        sat_table.append_column(get_hk_pk_column())
+        sat_table.append_column(get_loaddate_pk_column())
+        sat_table.append_column(get_recordsource_column())
+        sat_table.append_column(get_hashdiff_column())
+        for business_col in get_business_columns():
+            sat_table.append_column(business_col)
+        sat_table.append_column(get_is_available_column())
+
+        # Create dim table
+        dim_table = Table(entity_name, MetaData(), schema="dim")
+        for bk_col in get_bk_pk_columns():
+            dim_table.append_column(bk_col)
+        for business_col in get_business_columns():
+            dim_table.append_column(business_col)
+        dim_table.append_column(get_is_available_column())
+        dim_table.append_column(Column("IsCurrent", self.dialect_handler.get_boolean_type(), nullable=False))
+        dim_table.append_column(get_datefrom_pk_column())
+        dim_table.append_column(get_dateto_column())
+
+        return {
+            "hub": hub_table,
+            "sat": sat_table,
+            "dim": dim_table,
+        }
+
+    def make_procedures(self, tables: dict[str, Table]
+                        , entity_registration_date: datetime = datetime.now()) -> dict[str, Tuple[str, str, str]]:
+        procedures = {}
+
+        header = modeling_metadata.HEADER_TEMPLATE.format(
+            created_on=entity_registration_date,
+            updated_on=datetime.now(),
+            version=SANDWICH_VERSION,
+            entity_name=self._validation_result.entity_name
+        )
+
+        stg_proc_name = None
+        if self._validation_result.stg_schema == "proxy":
+            stg_proc_code, stg_proc_name, stg_call_stmt = self.dialect_handler.make_stg_materialization_proc(
+                entity_name=self._validation_result.entity_name,
+                header=header
+            )
+            procedures["stg"] = (stg_proc_code, stg_proc_name, stg_call_stmt)
+
+        hub_table = tables["hub"]
+        hub_proc_code, hub_proc_name, hub_call_stmt = self.dialect_handler.make_hub_proc(
+            hub_table=hub_table,
+            bk_keys=self._validation_result.bk_keys,
+            header=header
+        )
+        procedures["hub"] = (hub_proc_code, hub_proc_name, hub_call_stmt)
+
+        # Generate sat procedure
+        sat_table = tables["sat"]
+        sat_proc_code, sat_proc_name, sat_call_stmt = self.dialect_handler.make_scd2_sat_proc(
+            sat_table=sat_table,
+            hk_name=self._validation_result.hk_keys[0][0],
+            hashdiff_col=modeling_metadata.hashdiff,
+            is_available_col=modeling_metadata.is_available,
+            loaddate_col=modeling_metadata.loaddate,
+            stg_schema=self._validation_result.stg_schema,
+            header=header
+        )
+        procedures["sat"] = (sat_proc_code, sat_proc_name, sat_call_stmt)
+
+        # Generate dim procedure
+        dim_table = tables["dim"]
+        dim_proc_code, dim_proc_name, dim_call_stmt = self.dialect_handler.make_scd2_dim_proc(
+            dim_table=dim_table,
+            bk_keys=self._validation_result.bk_keys,
+            header=header
+        )
+        procedures["dim"] = (dim_proc_code, dim_proc_name, dim_call_stmt)
+
+        # Generate job procedure
+        job_proc_names = [] # order-sensitive
+        if self._validation_result.stg_schema == "proxy":
+            job_proc_names.append(stg_proc_name)
+        job_proc_names.extend([hub_proc_name, sat_proc_name, dim_proc_name])
+        job_proc_code, job_proc_name, job_call_stmt = self.dialect_handler.make_job_proc(
+            entity_name=self._validation_result.entity_name,
+            proc_names=job_proc_names,
+            header=header
+        )
+        procedures["job"] = (job_proc_code, job_proc_name, job_call_stmt)
+
+        # Generate drop procedure
+        drop_table_schemas = ["hub", "sat", "dim"]
+        if self._validation_result.stg_schema == "proxy":
+            drop_table_schemas.append("stg")
+        drop_proc_names = [job_proc_name, hub_proc_name, sat_proc_name, dim_proc_name]
+        if self._validation_result.stg_schema == "proxy":
+            drop_proc_names.append(stg_proc_name)
+        drop_proc_code, drop_proc_name, drop_call_stmt = self.dialect_handler.make_drop_proc(
+            entity_name=self._validation_result.entity_name,
+            table_schemas=drop_table_schemas,
+            procedures=drop_proc_names,
+            header=header
+        )
+        procedures["drop"] = (drop_proc_code, drop_proc_name, drop_call_stmt)
+
+        return procedures