sandwich 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

sandwich/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+SANDWICH_VERSION = version("sandwich")

sandwich/dialects/__init__.py

@@ -0,0 +1,12 @@
+"""Dialects package for SQL code generation."""
+from src.sandwich.dialects.base import DialectHandler
+from src.sandwich.dialects.factory import DialectHandlerFactory
+from src.sandwich.dialects.mssql import MssqlDialectHandler
+from src.sandwich.dialects.postgres import PostgresDialectHandler
+
+__all__ = [
+    "DialectHandler",
+    "DialectHandlerFactory",
+    "MssqlDialectHandler",
+    "PostgresDialectHandler",
+]

sandwich/dialects/base.py

@@ -0,0 +1,136 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from sqlalchemy import Table, TextClause
+
+class DialectHandler(ABC):
+    @abstractmethod
+    def get_boolean_type(self): ...
+
+    @abstractmethod
+    def get_proc_name_format(self, schema: str, operation: str, entity_name: str) -> str:
+        pass
+
+    @abstractmethod
+    def apply_proc_template(self, proc_name: str, sql_body: str, header: str) -> str:
+        """Wrap SQL body in procedure template with error handling and logging.
+
+        Args:
+            proc_name: Name of the procedure
+            sql_body: The main SQL logic to execute
+            header: Auto-generated header comment
+
+        Returns:
+            Complete procedure definition
+        """
+        pass
+
+    @abstractmethod
+    def make_stg_materialization_proc(
+        self,
+        entity_name: str,
+        header: str
+    ) -> Tuple[str, str, str]:
+        """Generate staging table materialization procedure.
+
+        Args:
+            entity_name: Entity name
+            columns_list: Comma-separated list of columns
+            header: Auto-generated header comment
+
+        Returns:
+            Tuple of (procedure_code, procedure_name)
+        """
+        pass
+
+    @abstractmethod
+    def make_hub_proc(
+        self,
+        hub_table: Table,
+        bk_keys: list,
+        header: str
+    ) -> Tuple[str, str, str]:
+        """Generate hub population procedure.
+
+        Args:
+            hub_table: SQLAlchemy Table object for hub
+            bk_keys: List of business key tuples (name, type)
+            columns_list: Comma-separated list of columns
+            header: Auto-generated header comment
+
+        Returns:
+            Tuple of (procedure_code, procedure_name)
+        """
+        pass
+
+    @abstractmethod
+    def make_link_proc(
+        self,
+        link_table: Table,
+        hk_keys: list,
+        header: str
+    ) -> Tuple[str, str, str]:
+        pass
+
+    @abstractmethod
+    def make_scd2_sat_proc(
+        self,
+        sat_table: Table,
+        hk_name: str,
+        hashdiff_col: str,
+        is_available_col: str,
+        loaddate_col: str,
+        stg_schema: str,
+        header: str
+    ) -> Tuple[str, str, str]:
+        pass
+
+    @abstractmethod
+    def make_scd0_sat_proc(self, sat_table: Table, header: str) -> Tuple[str, str, str]:
+        pass
+
+    @abstractmethod
+    def make_scd2_dim_proc(
+        self,
+        dim_table: Table,
+        bk_keys: list,
+        header: str
+    ) -> Tuple[str, str, str]:
+        """Generate dimension SCD2 recalculation procedure.
+
+        Args:
+            dim_table: SQLAlchemy Table object for dimension
+            bk_keys: List of business key tuples (name, type)
+            columns_list: Comma-separated list of columns
+            header: Auto-generated header comment
+
+        Returns:
+            Tuple of (procedure_code, procedure_name)
+        """
+        pass
+
+    @abstractmethod
+    def make_job_proc(
+        self,
+        entity_name: str,
+        proc_names: list[str],
+        header: str
+    ) -> Tuple[str, str, str]:
+        """Generate main job orchestration procedure.
+
+        Args:
+            entity_name: Entity name
+            hub_proc_name: Name of hub population procedure
+            sat_proc_name: Name of satellite population procedure
+            dim_proc_name: Name of dimension recalculation procedure
+            stg_proc_name: Name of staging materialization procedure (optional)
+            header: Auto-generated header comment
+
+        Returns:
+            Tuple of (procedure_code, procedure_name)
+        """
+        pass
+
+    @abstractmethod
+    def make_drop_proc(self, entity_name, table_schemas: list[str], procedures: list[str], header: str) \
+        -> Tuple[str, str, str]: ...

sandwich/dialects/ddl_mssql.py

@@ -0,0 +1,123 @@
+# language=sql
+create_entities_table = """
+if object_id('core.entities') is null
+begin
+    create table core.entities (
+        [entity_id] bigint primary key identity,
+        [entity_name] varchar(100) not null,
+        [template] varchar(50) not null,
+        [created] datetime2(7) not null default sysdatetime(),
+        [updated] datetime2(7) not null default sysdatetime(),
+        [is_deleted] bit not null default 0,
+        [deleted] datetime2(7) default null,
+        unique ([entity_name])
+    );
+end
+"""
+
+# language=sql
+create_table_ExecutionLog = """
+if object_id('core.ExecutionLog') is null
+begin
+    create table [core].[ExecutionLog](
+        [executionID] [bigint] identity(1,1) primary key NOT NULL,
+        [procid] [int] NOT NULL,
+        [begin_timestamp] [datetime2](7) NOT NULL default (getdate()),
+        [end_timestamp] [datetime2](7) NULL default (NULL),
+        [errorID] [int] NULL,
+        [procname] [varchar](200) NULL,
+        [parent_executionID] [bigint] NULL
+    )
+end
+"""
+
+# language=sql
+create_table_ErrorLog = """
+if object_id('core.ErrorLog') is null
+begin
+    create table [core].[ErrorLog](
+        [ErrorID] [int] IDENTITY(1,1) NOT NULL,
+        [UserName] [varchar](100) NULL,
+        [ErrorNumber] [int] NULL,
+        [ErrorState] [int] NULL,
+        [ErrorSeverity] [int] NULL,
+        [ErrorLine] [int] NULL,
+        [ErrorProcedure] [varchar](max) NULL,
+        [ErrorMessage] [varchar](max) NULL,
+        [ErrorDateTime] [datetime] NULL
+    );
+end
+"""
+
+# language=sql
+create_func_StringToHash = """
+create or alter function [core].[StringToHash1]
+(
+	@StrValue1 nvarchar(1000)
+) returns char(40) as
+begin
+	declare @result char(40);
+	set @result = upper(convert(char(40), hashbytes('sha1',
+		upper(rtrim(ltrim(isnull(@StrValue1, ''))))
+	), 2));
+	return @result;
+end
+"""
+
+# language=sql
+create_schemas = """
+if schema_id('core') is null
+    exec ('create schema core')
+if schema_id('stg') is null
+    exec ('create schema stg')
+if schema_id('hub') is null
+    exec ('create schema hub')
+if schema_id('sat') is null
+    exec ('create schema sat')
+if schema_id('link') is null
+    exec ('create schema link')
+if schema_id('dim') is null
+    exec ('create schema dim')
+if schema_id('fact') is null
+    exec ('create schema fact')
+if schema_id('elt') is null
+    exec ('create schema elt')
+if schema_id('job') is null
+    exec ('create schema job')
+if schema_id('meta') is null
+    exec ('create schema meta')
+if schema_id('proxy') is null
+    exec ('create schema proxy')
+"""
+
+# language=sql
+create_proc_LogExecution = """
+create or alter proc [core].[LogExecution]
+(
+	@procid int,
+	@executionID_in bigint,
+	@executionID_out bigint out,
+	@parent_executionID bigint = null
+) as
+begin
+	set nocount on;
+
+	if @executionID_in is not null
+	begin
+		update [core].[ExecutionLog]
+		set [end_timestamp] = getdate()
+		where executionID = @executionID_in;
+
+		set @executionID_out = @executionID_in;
+	end else
+	begin
+
+		declare @out table (executionID int);
+		insert into [core].[ExecutionLog] (procid, procname, parent_executionID) output inserted.executionID
+		into @out
+		values (@procid, object_name(@procid), @parent_executionID);
+
+		set @executionID_out = (select executionID from @out);
+	end
+end
+"""

sandwich/dialects/ddl_postgres.py

@@ -0,0 +1,114 @@
+# language=sql
+create_extensions = """
+create extension if not exists pgcrypto;
+"""
+
+# language=sql
+create_entities_table = """
+CREATE TABLE IF NOT EXISTS core.entities (
+    entity_id BIGSERIAL PRIMARY KEY,
+    entity_name VARCHAR(100) NOT NULL,
+    template VARCHAR(50) NOT NULL,
+    created TIMESTAMP NOT NULL DEFAULT NOW(),
+    updated TIMESTAMP NOT NULL DEFAULT NOW(),
+    is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
+    deleted TIMESTAMP DEFAULT NULL,
+    UNIQUE (entity_name)
+);
+"""
+
+# language=sql
+create_func_StringToHash= """
+create or replace function core.string_to_hash1(str_value text)
+returns char(40)
+language plpgsql
+as $$
+declare
+    result char(40);
+begin
+    if str_value is null
+       or str_value in ('(unknown)', 'empty')
+    then
+        result := repeat('0', 40);
+    else
+        result :=
+            upper(
+                encode(
+                    digest(upper(trim(str_value)), 'sha1'),
+                    'hex'
+                )
+            );
+    end if;
+
+    return cast(result as char(40));
+end;
+$$;
+"""
+
+# language=sql
+create_table_ExecutionLog = """
+CREATE TABLE IF NOT EXISTS core.ExecutionLog (
+    executionID BIGSERIAL PRIMARY KEY,
+    procid INT NOT NULL,
+    begin_timestamp TIMESTAMP NOT NULL DEFAULT NOW(),
+    end_timestamp TIMESTAMP DEFAULT NULL,
+    errorID INT DEFAULT NULL,
+    procname VARCHAR(200) DEFAULT NULL,
+    parent_executionID BIGINT DEFAULT NULL
+);
+"""
+
+# language=sql
+create_table_ErrorLog = """
+CREATE TABLE IF NOT EXISTS core.ErrorLog (
+    ErrorID SERIAL PRIMARY KEY,
+    UserName VARCHAR(100) DEFAULT NULL,
+    ErrorNumber INT DEFAULT NULL,
+    ErrorState INT DEFAULT NULL,
+    ErrorSeverity INT DEFAULT NULL,
+    ErrorLine INT DEFAULT NULL,
+    ErrorProcedure TEXT DEFAULT NULL,
+    ErrorMessage TEXT DEFAULT NULL,
+    ErrorDateTime TIMESTAMP DEFAULT NULL
+);
+"""
+
+# language=sql
+create_proc_LogExecution = """
+CREATE OR REPLACE PROCEDURE core.LogExecution(
+    p_procid INT,
+    p_executionID_in BIGINT,
+    INOUT p_executionID_out BIGINT,
+    p_parent_executionID BIGINT DEFAULT NULL
+)
+LANGUAGE plpgsql
+AS $$
+BEGIN
+    IF p_executionID_in IS NOT NULL THEN
+        UPDATE core.ExecutionLog
+        SET end_timestamp = NOW()
+        WHERE executionID = p_executionID_in;
+
+        p_executionID_out := p_executionID_in;
+    ELSE
+        INSERT INTO core.ExecutionLog (procid, procname, parent_executionID)
+        VALUES (p_procid, NULL, p_parent_executionID)
+        RETURNING executionID INTO p_executionID_out;
+    END IF;
+END;
+$$;
+"""
+
+# language=sql
+create_schemas = f"""
+CREATE SCHEMA IF NOT EXISTS core;
+CREATE SCHEMA IF NOT EXISTS stg;
+CREATE SCHEMA IF NOT EXISTS hub;
+CREATE SCHEMA IF NOT EXISTS sat;
+CREATE SCHEMA IF NOT EXISTS dim;
+CREATE SCHEMA IF NOT EXISTS fact;
+CREATE SCHEMA IF NOT EXISTS elt;
+CREATE SCHEMA IF NOT EXISTS job;
+CREATE SCHEMA IF NOT EXISTS meta;
+CREATE SCHEMA IF NOT EXISTS proxy;
+"""

sandwich/dialects/factory.py

@@ -0,0 +1,27 @@
+from src.sandwich.dialects.base import DialectHandler
+from src.sandwich.dialects.mssql import MssqlDialectHandler
+from src.sandwich.dialects.postgres import PostgresDialectHandler
+
+
+class DialectHandlerFactory:
+    _handlers = {
+        "mssql": MssqlDialectHandler,
+        "postgres": PostgresDialectHandler,
+    }
+
+    @classmethod
+    def register_dialect(cls, dialect_name: str, handler_class):
+        cls._handlers[dialect_name] = handler_class
+
+    @classmethod
+    def create_handler(cls, dialect: str) -> DialectHandler:
+        if dialect not in cls._handlers:
+            available = ", ".join(cls._handlers.keys())
+            raise ValueError(f"Unknown dialect '{dialect}'. Available dialects: {available}")
+
+        handler_class = cls._handlers[dialect]
+        return handler_class()
+
+    @classmethod
+    def get_available_dialects(cls) -> list[str]:
+        return list(cls._handlers.keys())

sandwich/dialects/mssql.py

@@ -0,0 +1,281 @@
+"""MSSQL dialect handler for SQL code generation."""
+from typing import Tuple
+
+from sqlalchemy import dialects, Table, text
+
+#from sandwich import SANDWICH_VERSION
+#from sandwich.modeling import modeling_metadata
+#from sandwich.modeling.strategies.base import ValidationResult
+
+from .base import DialectHandler
+from .utils import get_columns_list
+
+class MssqlDialectHandler(DialectHandler):
+    """Dialect handler for Microsoft SQL Server."""
+
+    def get_boolean_type(self):
+        return dialects.mssql.BIT
+
+    def get_proc_name_format(self, schema: str, operation: str, entity_name: str) -> str:
+        """Get MSSQL procedure naming format."""
+        return f"[{schema}].[{operation}_{entity_name}]"
+
+    def apply_proc_template(self, proc_name: str, sql_body: str, header: str) -> str:
+        """Wrap SQL body in MSSQL procedure template with error handling."""
+        # language=sql
+        proc_template_sql = f"""
+{header}
+create or alter proc {proc_name} (@parent_executionID bigint = null) as
+begin
+    set nocount on;
+
+    declare @executionID bigint;
+    exec core.LogExecution @@PROCID, null, @executionID out, @parent_executionID;
+
+begin try
+    {sql_body}
+    exec core.LogExecution @@PROCID, @executionID, @executionID out;
+end try
+begin catch
+    declare @err table (ErrorID int);
+    declare @ErrorMessage NVARCHAR(4000);
+    declare @ErrorSeverity INT;
+    declare @ErrorState INT;
+
+    set @ErrorMessage = ERROR_MESSAGE();
+    set @ErrorSeverity = ERROR_SEVERITY();
+    set @ErrorState = ERROR_STATE();
+
+    insert into core.ErrorLog
+    output inserted.ErrorID into @err
+    values (
+        SUSER_SNAME(),
+        ERROR_NUMBER(),
+        @ErrorState,
+        @ErrorSeverity,
+        ERROR_LINE(),
+        ERROR_PROCEDURE(),
+        @ErrorMessage,
+        getdate()
+    );
+
+    update [core].[ExecutionLog]
+    set [errorID] = (select ErrorID from @err)
+    , [end_timestamp] = getdate()
+    where [executionID] = @executionID;
+
+    RAISERROR (
+        @ErrorMessage,
+        @ErrorSeverity,
+        @ErrorState
+    );
+end catch
+end
+"""
+        return proc_template_sql
+
+    def make_stg_materialization_proc(self, entity_name: str, header: str) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Populate_stg", entity_name)
+
+        # language=sql
+        proc_body = f"""
+    if object_id('stg.{entity_name}') is not null drop table stg.{entity_name};
+    select *
+    into stg.{entity_name}
+    from proxy.{entity_name};
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_hub_proc(self, hub_table: Table, bk_keys: list, header: str) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Populate_{hub_table.schema}", hub_table.name)
+        where_fields_list_str = " and ".join([f"hub.[{bk[0]}] = stg.[{bk[0]}]" for bk in bk_keys])
+        columns_list = get_columns_list(hub_table)
+
+        # language=sql
+        proc_body = f"""
+    insert into [{hub_table.schema}].[{hub_table.name}]
+    ({columns_list})
+    select distinct {get_columns_list(hub_table, alias="stg")}
+    from stg.[{hub_table.name}] as stg
+    where not exists (
+        select *
+        from [{hub_table.schema}].[{hub_table.name}] as hub
+        where {where_fields_list_str}
+    );
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_link_proc(self, link_table: Table, hk_keys: list, header: str) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Populate_{link_table.schema}", link_table.name)
+        where_fields_list_str = "\n\t\tand ".join([f"link.[{hk[0]}] = stg.[{hk[0]}]" for hk in hk_keys if hk[0] != f"hk_{link_table.name}"])
+        columns_list = get_columns_list(link_table)
+
+        # language=sql
+        proc_body = f"""
+    insert into [{link_table.schema}].[{link_table.name}]
+    ({columns_list})
+    select distinct {get_columns_list(link_table, alias="stg")}
+    from stg.[{link_table.name}] as stg
+    where not exists (
+        select *
+        from [{link_table.schema}].[{link_table.name}] as link
+        where {where_fields_list_str}
+    );
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_scd0_sat_proc(self, sat_table: Table, header: str) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Populate_{sat_table.schema}", sat_table.name)
+        columns_list = get_columns_list(sat_table)
+        hk_name = f"hk_{sat_table.name}"
+
+        # language=sql
+        proc_body = f"""
+    insert into [{sat_table.schema}].[{sat_table.name}]
+    ({columns_list})
+    select {get_columns_list(sat_table, alias="stg")}
+    from stg.[{sat_table.name}] stg
+    where not exists (
+        select *
+        from sat.[{sat_table.name}] sat
+        where stg.[{hk_name}] = sat.[{hk_name}]
+    )
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_scd2_sat_proc(
+        self,
+        sat_table: Table,
+        hk_name: str,
+        hashdiff_col: str,
+        is_available_col: str,
+        loaddate_col: str,
+        stg_schema: str,
+        header: str
+    ) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Populate_{sat_table.schema}", sat_table.name)
+        columns_list = get_columns_list(sat_table)
+
+        def smart_replace(column_name: str) -> str:
+            if column_name == "LoadDate":
+                result = "sysdatetime() as [LoadDate]"
+            elif column_name == "IsAvailable":
+                result = "cast(0 as bit) as [IsAvailable]"
+            else:
+                result = f"sat.[{column_name}]"
+            return result
+
+        select_columns_list = ", ".join([smart_replace(col.name) for col in sat_table.columns.values()])
+
+        if stg_schema == "proxy":
+            stg_table_name = f"stg.[{sat_table.name}]"
+            materialization_stmt = ""
+        else:
+            stg_table_name = "#materialized"
+            materialization_stmt = f"""
+    select distinct {columns_list}
+    into #materialized
+    -- drop table #materialized
+    from stg.[{sat_table.name}];
+"""
+
+        # language=sql
+        proc_body = f"""{materialization_stmt}
+    with ranked_history as
+    (
+        select {columns_list}
+        , row_number() over (partition by [{hk_name}] order by [{loaddate_col}] desc) [DescRank]
+        from [{sat_table.schema}].[{sat_table.name}]
+    )
+    insert into [{sat_table.schema}].[{sat_table.name}]
+    ({columns_list})
+    select {get_columns_list(sat_table, alias="stg")}
+    from {stg_table_name} stg
+    where not exists (
+        select *
+        from ranked_history sat
+        where sat.[DescRank] = 1
+        and stg.[{hk_name}] = sat.[{hk_name}]
+        and stg.[{hashdiff_col}] = sat.[{hashdiff_col}]
+        and sat.[{is_available_col}] = 1
+    )
+
+    union all
+
+    select {select_columns_list}
+    from ranked_history sat
+    where not exists (
+        select *
+        from {stg_table_name} stg
+        where stg.[{hk_name}] = sat.[{hk_name}]
+    )
+    and sat.[DescRank] = 1
+    and sat.[{is_available_col}] = 1;
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_scd2_dim_proc(self, dim_table: Table, bk_keys: list, header: str) -> Tuple[str, str, str]:
+        proc_name = self.get_proc_name_format("elt", f"Recalculate_{dim_table.schema}", dim_table.name)
+        columns_list = get_columns_list(dim_table)
+        pk_keys = lambda: ", ".join([f"sat.[{bk[0]}]" for bk in bk_keys])
+
+        def smart_replace(column_name: str) -> str:
+            if column_name == "DateFrom":
+                result = "sat.LoadDate as [DateFrom]"
+            elif column_name == "DateTo":
+                result = f"lead(dateadd(microsecond, -1, sat.LoadDate), 1, '9999-12-31 23:59:59.9999999') over (partition by {pk_keys()} order by sat.LoadDate) [DateTo]"
+            elif column_name == "IsCurrent":
+                result = f"iif(lead(sat.LoadDate) over (partition by {pk_keys()} order by sat.LoadDate) is null, 1, 0) [IsCurrent]"
+            else:
+                result = f"sat.[{column_name}]"
+            return result
+
+        select_columns_list = "\n\t, ".join([smart_replace(col.name) for col in dim_table.columns.values()])
+
+        # language=sql
+        proc_body = f"""
+    truncate table [{dim_table.schema}].[{dim_table.name}];
+
+    insert into [{dim_table.schema}].[{dim_table.name}]
+    ({columns_list})
+    select {select_columns_list}
+    from sat.[{dim_table.name}] sat
+"""
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_job_proc(self, entity_name: str, proc_names: list[str], header: str) -> Tuple[str, str, str]:
+        proc_name = f"[job].[Run_all_related_to_{entity_name}]"
+        proc_body = "\n\t"
+        for proc in proc_names:
+            if proc is None: continue
+            proc_body += f"exec {proc} @executionID;\n\t"
+
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+
+    def make_drop_proc(self, entity_name, table_schemas: list[str], procedures: list[str], header: str) -> Tuple[str, str, str]:
+        proc_name = f"[meta].[Drop_all_related_to_{entity_name}]"
+        proc_body = "\n\t"
+        for proc in procedures:
+            if proc is None: continue
+            proc_body += f"drop procedure if exists {proc};\n\t"
+        proc_body += "\n\t"
+        for schema in table_schemas:
+            proc_body += f"drop table if exists [{schema}].[{entity_name}];\n\t"
+        proc_body += "\n\t"
+        proc_body += \
+f"""update core.[entities]
+    set [deleted] = sysdatetime()
+    , [is_deleted] = 1
+    where [entity_name] = '{entity_name}'
+"""
+
+        proc_code = self.apply_proc_template(proc_name, proc_body, header)
+        return proc_code, proc_name, f"exec {proc_name}"
+