icsDataValidation 1.0.361__py3-none-any.whl → 1.0.365__py3-none-any.whl

examples/ics_data_validation.py

@@ -0,0 +1,7 @@
+
+from  manual_execution_params import manual_execution_params
+from icsDataValidation import main
+
+manual_execution_params()
+
+main.execute()

examples/manual_execution_params.template.py

@@ -0,0 +1,44 @@
+#########################################################################################
+#########################################################################################
+import os
+
+def manual_execution_params():
+
+    # Manual execution: File location of the icsDataValidation configuration
+    os.environ["CONFIG_FOLDER_NAME"]                    = 'examples/'
+    os.environ["CONFIGURATION_FILE_NAME"]               = 'ics_data_validation_config.json'
+    os.environ["MIGRATION_CONFIGURATION_FILE_NAME"]     = 'migration_config.json'
+
+    # Manual execution: File path of the locally stored secrets
+    # Syntax: <parameter_name>="<value>" per row
+    os.environ["ENV_FILEPATH"]  = ''
+
+    # Manual execution: Testset settings
+    os.environ["DATABASE_NAME"] = '' #
+    os.environ["SCHEMA_NAME"] = '' #
+
+    os.environ["TESTSET_FILE_NAMES"] = ''  # for no testset define as ''
+
+    os.environ["OBJECT_TYPE_RESTRICTION"] = '' #'include_all', 'include_only_tables', 'include_only_views'
+
+    # Manual execution: Result settings
+    os.environ["UPLOAD_RESULT_TO_BLOB"] = '' #boolean: True or False
+    os.environ["UPLOAD_RESULT_TO_BUCKET"] = '' #boolean: True or False
+    os.environ["UPLOAD_RESULT_TO_RESULT_DATABASE"] = ''#boolean: True or False
+
+    # Manual execution: Pandas Dataframe Comparison restrictions -> -1 for no pandas-df comparison at all
+    os.environ["MAX_OBJECT_SIZE"] = str(-1) #-1
+    os.environ["MAX_ROW_NUMBER"]  = str(-1) #-1
+
+    # Manual execution: Parallelization of comparison settings
+    os.environ["MAX_NUMBER_OF_THREADS"]  = str(1) #1
+
+    # Manual execution: Group-By-Aggregation settings
+    os.environ["EXECUTE_GROUP_BY_COMPARISON"] = '' #boolean: True or False
+    os.environ["USE_GROUP_BY_COLUMNS"] = '' #boolean: True or False
+    os.environ["MIN_GROUP_BY_COUNT_DISTINCT"] = str(2) #2
+    os.environ["MAX_GROUP_BY_COUNT_DISTINCT"] = str(5) #5
+    os.environ["MAX_GROUP_BY_SIZE"] = str(100000000) #100000000
+
+    # Manual execution: Precision settings
+    os.environ["NUMERIC_SCALE"] = str(2)

icsDataValidation/connection_setups/snowflake_connection_setup.py

@@ -1,35 +1,20 @@
 import os
 
-from cloe_util_snowflake_connector.connection_parameters import ConnectionParameters, EnvVariablesInitializer
+from dotenv import load_dotenv
+from pathlib import Path
 
 #########################################################################################
 #########################################################################################
 
+def load_snowflake_credentials(system_configs:dict,system_selection:str)->dict:
 
-def load_snowflake_credentials(system_configs: dict, system_selection: str) -> ConnectionParameters:
-    snowflake_params = EnvVariablesInitializer(
-        user=system_configs[system_selection]["USER"],
-        account=system_configs[system_selection]["ACCOUNT"],
-        warehouse=system_configs[system_selection]["WAREHOUSE"],
-        database=system_configs[system_selection]["DATABASE"],
-        role=system_configs[system_selection]["ROLE"],
-        password=os.getenv(system_configs[system_selection]["PASSWORD_NAME"])
-        if "PASSWORD_NAME" in system_configs[system_selection]
-        else None,
-        private_key=os.getenv(system_configs[system_selection]["PRIVATE_KEY_NAME"])
-        if "PRIVATE_KEY_NAME" in system_configs[system_selection]
-        else None,
-        private_key_passphrase=os.getenv(system_configs[system_selection]["PRIVATE_KEY_PASSPHRASE_NAME"])
-        if "PRIVATE_KEY_PASSPHRASE_NAME" in system_configs[system_selection]
-        else None,
-        private_key_file=os.getenv(system_configs[system_selection]["PRIVATE_KEY_FILE_PATH"])
-        if "PRIVATE_KEY_FILE_PATH" in system_configs[system_selection]
-        else None,
-        private_key_file_pwd=os.getenv(system_configs[system_selection]["PRIVATE_KEY_FILE_PASSWORD"])
-        if "PRIVATE_KEY_FILE_PASSWORD" in system_configs[system_selection]
-        else None,
-    )
+    snowflake_params = {
+        "account"   : system_configs[system_selection]["ACCOUNT"],
+        "user"      : system_configs[system_selection]["USER"],
+        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"]),
+        "warehouse" : system_configs[system_selection]["WAREHOUSE"],
+        "role"      : system_configs[system_selection]["ROLE"],
+        "database"  : system_configs[system_selection]["DATABASE"]
+    }
 
-    connection_params = ConnectionParameters(**snowflake_params.model_dump())
-
-    return connection_params
+    return snowflake_params

icsDataValidation/services/comparison_service.py

@@ -4,7 +4,7 @@ import datetime
 import numpy as np
 
 from pandas._testing import assert_frame_equal
-from decimal import Decimal
+from decimal import Decimal, InvalidOperation, getcontext
 
 from icsDataValidation.utils.logger_util import configure_dev_ops_logger
 from icsDataValidation.utils.pandas_util import get_diff_dataframes, get_diff_dict_from_diff_dataframes
@@ -166,16 +166,31 @@ class ComparisonService(TestingToolParams):
         del trgt_columns_aggregate['TESTATM_ERRORS']
 
         if self.result_params.src_row_count != 0 and self.result_params.trgt_row_count != 0:
-            aggregation_differences_trgt_minus_src_not_boolean = {
-                                                                    k:  round(Decimal(trgt_columns_aggregate[k][1]) 
-                                                                        - Decimal(src_columns_aggregate[k][1]), self.numeric_scale) 
-                                                                    for k in src_columns_aggregate.keys() 
-                                                                        if k in trgt_columns_aggregate 
-                                                                        and str(src_columns_aggregate[k][1]) != str(trgt_columns_aggregate[k][1]) 
-                                                                        and src_columns_aggregate[k][1] != trgt_columns_aggregate[k][1]
-                                                                        and src_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
-                                                                        and trgt_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
-                                                                }
+            try:
+                aggregation_differences_trgt_minus_src_not_boolean = {
+                                                                        k:  round(Decimal(trgt_columns_aggregate[k][1]) 
+                                                                            - Decimal(src_columns_aggregate[k][1]), self.numeric_scale) 
+                                                                        for k in src_columns_aggregate.keys() 
+                                                                            if k in trgt_columns_aggregate 
+                                                                            and str(src_columns_aggregate[k][1]) != str(trgt_columns_aggregate[k][1]) 
+                                                                            and src_columns_aggregate[k][1] != trgt_columns_aggregate[k][1]
+                                                                            and src_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
+                                                                            and trgt_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
+                                                                    }
+            except InvalidOperation as e:
+                getcontext().prec = 100 # sets the precision of Decimal to a higher value - due to the limitations of the decimal module when handling such large numbers with high precision
+                aggregation_differences_trgt_minus_src_not_boolean = {
+                                                                        k:  round(Decimal(trgt_columns_aggregate[k][1]) 
+                                                                            - Decimal(src_columns_aggregate[k][1]), self.numeric_scale) 
+                                                                        for k in src_columns_aggregate.keys() 
+                                                                            if k in trgt_columns_aggregate 
+                                                                            and str(src_columns_aggregate[k][1]) != str(trgt_columns_aggregate[k][1]) 
+                                                                            and src_columns_aggregate[k][1] != trgt_columns_aggregate[k][1]
+                                                                            and src_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
+                                                                            and trgt_columns_aggregate[k][0].upper() != 'AGGREGATEBOOLEAN'
+                                                                    }
+                
+                
             aggregation_differences_trgt_minus_src_boolean = {
                                                                 k:  str(
                                                                         int(trgt_columns_aggregate[k][1].split('_',1)[0]) 

icsDataValidation/services/database_services/snowflake_service.py

@@ -1,33 +1,34 @@
-import logging
-from pathlib import PurePath
 
-import pandas as pd
 import snowflake.connector
-from cloe_util_snowflake_connector import connection_parameters, snowflake_interface
+import pandas as pd
+import logging
+
+from typing import Union, List, Dict 
+from pathlib import PurePath
 
-from icsDataValidation.core.database_objects import DatabaseObject
 from icsDataValidation.utils.logger_util import configure_dev_ops_logger
+from icsDataValidation.core.database_objects import DatabaseObject
 
 #########################################################################################
 #########################################################################################
 
 # Configure Dev Ops Logger
 
-logger = logging.getLogger("Snowflake_Service")
+logger = logging.getLogger('Snowflake_Service')
 logger.setLevel(logging.INFO)
 configure_dev_ops_logger(logger)
 
+class SnowflakeService(object):
 
-class SnowflakeService:
-    def __init__(self, connection_params: connection_parameters.ConnectionParameters):
-        self.connection_params = connection_params
+    def __init__(self, connection_params: dict):
+        self.connection_params =connection_params
         self.snowflake_connection = None
         self.snowflake_datatype_mapping = {
-            "string": ["text"],
-            "numeric": ["number", "float"],
-            "date_and_time": ["date", "time", "timestamp_ntz", "timestamp_tz", "timestamp_ltz"],
-            "binary": ["binary"],
-            "boolean": ["boolean"],
+                                            "string": ['text'],
+                                            "numeric": ['number', 'float'],
+                                            "date_and_time" : ['date', 'time', 'timestamp_ntz', 'timestamp_tz', 'timestamp_ltz'],
+                                            "binary" : ['binary'],
+                                            "boolean" : ['boolean']
         }
 
     def __enter__(self):
@@ -42,7 +43,7 @@ class SnowflakeService:
             self.snowflake_connection.close()
 
     def _connect_to_snowflake(self):
-        self.snowflake_connection = snowflake_interface.SnowflakeInterface(self.connection_params)
+        self.snowflake_connection = snowflake.connector.connect(**self.connection_params)
         return self.snowflake_connection
 
     @staticmethod
@@ -61,8 +62,8 @@ class SnowflakeService:
         return f"Snowflake ERROR: {message}\nFailed statement:\n{statement}"
 
     @staticmethod
-    def _get_in_clause(key_filters: list, numeric_columns: list, numeric_scale: int) -> str:
-        """generates in_clause from list ready to expand the where clause, numeric values are rounded
+    def _get_in_clause(key_filters:list, numeric_columns:list, numeric_scale:int) -> str:
+        """ generates in_clause from list ready to expand the where clause, numeric values are rounded
 
         Args:
             key_filters (list): list of given expected values
@@ -71,26 +72,26 @@ class SnowflakeService:
 
         Returns:
             str: in clause as string
-        """
-        values = list(key_filters.values())
+        """        
+        values = list(key_filters.values())  
         in_clause_values = "('"
         for j in range(len(values[0])):
             for value in values:
                 in_clause_values += str(value[j]) + "','"
             in_clause_values = in_clause_values[:-2] + "),('"
-        in_clause_values = in_clause_values[:-3] + ")"
+        in_clause_values = in_clause_values[:-3] + ')'
 
-        in_clause_cols = " AND (("
+        in_clause_cols = f" AND ((" 
         for key in key_filters.keys():
             if key in numeric_columns:
                 in_clause_cols += f"""ROUND({key.replace("'", "")},2)""" + ","
             else:
                 in_clause_cols += key.replace("'", "") + ","
         in_clause_cols = in_clause_cols[:-1] + ")"
-        in_clause = in_clause_cols + " in (" + in_clause_values + ")"
+        in_clause = in_clause_cols + " in ("  + in_clause_values + ")"                                  
         return in_clause
-
-    def _get_column_clause(self, column_list: list, columns_datatype: list, numeric_scale, key_columns) -> dict:
+    
+    def _get_column_clause(self, column_list: list, columns_datatype: list,  numeric_scale, key_columns) ->dict : 
         """
         Turns list of desired columns into a sql compatible string.
         Columns with a date or time data type are omitted.
@@ -103,25 +104,23 @@ class SnowflakeService:
 
         Returns:
             dict: _description_
-        """
-        column_intersecions_new = []
+        """        
+        column_intersecions_new = []           
         used_columns = []
         numeric_columns = []
         for column in column_list:
-            column_datatype = next(x for x in columns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
+            column_datatype=next(x for x in columns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
 
-            if column in key_columns or column_datatype.lower() not in self.snowflake_datatype_mapping["date_and_time"]:
-                if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
+            if column in  key_columns or not (column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]):
+                if column_datatype.lower() in  self.snowflake_datatype_mapping["numeric"]:
                     if numeric_scale:
-                        column_intersecions_new.append(
-                            f"CAST(ROUND({column}, {numeric_scale}) as decimal(38,{numeric_scale})) as {column}"
-                        )
+                        column_intersecions_new.append(f"CAST(ROUND({column}, {numeric_scale}) as decimal(38,{numeric_scale})) as {column}")
                     else:
                         column_intersecions_new.append(f"{column} as {column}")
                     used_columns.append(column)
                     numeric_columns.append(column)
-                elif column_datatype.lower() in self.snowflake_datatype_mapping["string"]:
-                    column_intersecions_new.append(f"{column} AS {column}")
+                elif column_datatype.lower() in  self.snowflake_datatype_mapping["string"]:
+                    column_intersecions_new.append(f'{column} AS {column}')
                     used_columns.append(column)
                 else:
                     column_intersecions_new.append(column)
@@ -131,43 +130,44 @@ class SnowflakeService:
         column_clause = str(column_intersections)[1:-1].replace("'", "")
         return column_clause, numeric_columns, used_columns
 
-    def get_database_objects(
-        self, database: str, schema: str = None, object_type_restriction: str = "include_all"
-    ) -> dict:
+    def get_database_objects(self, database: str, schema: str=None, object_type_restriction: str='include_all') -> dict:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        all_database_tables = []
-        all_database_views = []
+        all_database_tables=[]
+        all_database_views=[]
 
-        if object_type_restriction == "include_all" or object_type_restriction == "include_only_tables":
+        if object_type_restriction=='include_all' or object_type_restriction=='include_only_tables':
             if schema:
-                query_db_tables = f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE'; "
-            else:
-                query_db_tables = f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE';"
+                query_db_tables=f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE'; "
+            else:    
+                query_db_tables=f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE';"
 
             all_database_tables = self.execute_queries(query_db_tables)
 
-        if object_type_restriction == "include_all" or object_type_restriction == "include_only_views":
+
+        if object_type_restriction=='include_all' or object_type_restriction=='include_only_views':
             if schema:
-                query_db_views = f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
-            else:
-                query_db_views = (
-                    f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
-                )
+                query_db_views=f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
+            else:    
+                query_db_views=f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
 
             all_database_views = self.execute_queries(query_db_views)
+        
 
-        database_objects = []
+        database_objects=[]
         for row in all_database_tables:
-            table_identifier = f"{row['TABLE_CATALOG']}.{row['TABLE_SCHEMA']}.{row['TABLE_NAME']}"
+            table_identifier=f'{row["TABLE_CATALOG"]}.{row["TABLE_SCHEMA"]}.{row["TABLE_NAME"]}'
             database_objects.append({"object_identifier": table_identifier, "object_type": "table"})
         for row in all_database_views:
-            view_identifier = f"{row['TABLE_CATALOG']}.{row['TABLE_SCHEMA']}.{row['TABLE_NAME']}"
+            view_identifier=f'{row["TABLE_CATALOG"]}.{row["TABLE_SCHEMA"]}.{row["TABLE_NAME"]}'
             database_objects.append({"object_identifier": view_identifier, "object_type": "view"})
         return database_objects
 
-    def get_last_altered_timestamp_from_object(self, object: DatabaseObject) -> str:
+    def get_last_altered_timestamp_from_object(
+            self,
+            object: DatabaseObject
+        ) -> str:
         """queries last_altered timestamp for given object
 
         Args:
@@ -180,14 +180,14 @@ class SnowflakeService:
             self._connect_to_snowflake()
 
         self.execute_statement("ALTER SESSION SET TIMEZONE = 'Europe/London';")
-
-        query_get_last_altered = f"SELECT LAST_ALTERED FROM {object.database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{object.name}' AND TABLE_SCHEMA = '{object.schema}';"
+        
+        query_get_last_altered=f"SELECT LAST_ALTERED FROM {object.database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{object.name}' AND TABLE_SCHEMA = '{object.schema}';"
 
         last_altered = self.execute_queries(query_get_last_altered)[0]
 
         return last_altered
 
-    def get_columns_from_object(self, object: DatabaseObject) -> list:
+    def get_columns_from_object(self, object : DatabaseObject) -> list:
         """returns all columns from given object
 
         Args:
@@ -200,34 +200,30 @@ class SnowflakeService:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        if object.type == "table":
+        if object.type =='table':
             query_show_columns = f"SHOW COLUMNS IN TABLE {object.database}.{object.schema}.{object.name};"
 
-            show_columns_result, query_id, test = self.execute_queries(
-                query_show_columns, return_as_pdf=False, return_query_ids=True
-            )
-
+            show_columns_result, query_id, test = self.execute_queries(query_show_columns, return_as_pdf=False, return_query_ids=True)
+            
             query_get_columns = f"SELECT $3 AS COLUMN_NAME FROM TABLE(result_scan('{query_id}'));"
 
-        if object.type == "view":
+        if object.type =='view':
             query_show_columns = f"SHOW COLUMNS IN VIEW {object.database}.{object.schema}.{object.name};"
 
-            show_columns_result, query_id, test = self.execute_queries(
-                query_show_columns, return_as_pdf=False, return_query_ids=True
-            )
-
+            show_columns_result, query_id, test = self.execute_queries(query_show_columns, return_as_pdf=False, return_query_ids=True)
+            
             query_get_columns = f"SELECT $3 AS COLUMN_NAME FROM TABLE(result_scan('{query_id}'));"
 
         all_columns = self.execute_queries(query_get_columns)
-        columns = []
+        columns=[]
 
         for row in all_columns:
             columns.append(row["COLUMN_NAME"])
 
         return columns
 
-    def get_row_count_from_object(self, object: DatabaseObject, where_clause: str = "") -> int:
-        """gets row count from given object
+    def get_row_count_from_object(self, object : DatabaseObject, where_clause: str="") -> int:
+        """ gets row count from given object
 
         Args:
             object (DatabaseObject): table or view
@@ -238,25 +234,23 @@ class SnowflakeService:
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-
-        # TODO is it more efficient to select the information_schema.table view to get the rows?
-        query_get_row_count = (
-            f"SELECT COUNT(*) AS ROW_COUNT FROM {object.database}.{object.schema}.{object.name} {where_clause};"
-        )
+        
+        #TODO is it more efficient to select the information_schema.table view to get the rows?
+        query_get_row_count = f"SELECT COUNT(*) AS ROW_COUNT FROM {object.database}.{object.schema}.{object.name} {where_clause};"
         row_count = -1
         error_list = []
 
         try:
             row_count = self.execute_queries(query_get_row_count)[0]["ROW_COUNT"]
-
+        
         except Exception as err:
             error_list.append(str(err))
             error_list.append(query_get_row_count)
 
         return row_count, error_list
 
-    def get_data_types_from_object(self, object: DatabaseObject, column_intersections: list) -> dict:
-        """returns datatypes for all intersection columns in a database object
+    def get_data_types_from_object(self, object : DatabaseObject, column_intersections: list) -> dict:
+        """ returns datatypes for all intersection columns in a database object 
 
         Args:
             object (DatabaseObject): table or view
@@ -270,22 +264,20 @@ class SnowflakeService:
             self._connect_to_snowflake()
 
         column_intersections = str(column_intersections)[1:-1]
-        if column_intersections == "":
+        if column_intersections == '':
             column_intersections = "''"
 
-        query_get_data_types_from_object = f"SELECT COLUMN_NAME , DATA_TYPE   \
+        query_get_data_types_from_object=f"SELECT COLUMN_NAME , DATA_TYPE   \
                                             FROM {object.database.upper()}.INFORMATION_SCHEMA.COLUMNS  \
                                             WHERE TABLE_NAME='{object.name.upper()}'  \
                                             AND TABLE_SCHEMA = '{object.schema.upper()}'  \
                                             AND COLUMN_NAME IN ({column_intersections})  \
                                             ;"
 
-        dict_colummns_datatype = self.execute_queries(query_get_data_types_from_object)
+        dict_colummns_datatype=self.execute_queries(query_get_data_types_from_object)
         return dict_colummns_datatype
 
-    def get_count_distincts_from_object(
-        self, object: DatabaseObject, column_intersections: list, where_clause: str = "", exclude_columns: list = []
-    ) -> dict:
+    def get_count_distincts_from_object(self, object: DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns: list=[]) -> dict:
         """get distinct count for every column in a database object that is in column intersections list
 
         Args:
@@ -302,26 +294,27 @@ class SnowflakeService:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        unions = ""
+        unions=""
 
         for column in column_intersections:
             if column not in exclude_columns:
-                unions += f" UNION SELECT '{column}' AS COLUMN_NAME, COUNT(DISTINCT {column}) AS COUNT_DISTINCT FROM {object.database}.{object.schema}.{object.name} {where_clause}"
+                unions +=f" UNION SELECT '{column}' AS COLUMN_NAME, COUNT(DISTINCT {column}) AS COUNT_DISTINCT FROM {object.database}.{object.schema}.{object.name} {where_clause}"
 
-        query_get_count_distincts_from_object = f"{unions[6:]} ORDER BY COUNT_DISTINCT;"
+        query_get_count_distincts_from_object=f"{unions[6:]} ORDER BY COUNT_DISTINCT;"
         error_list = []
         try:
-            dict_count_distincts = self.execute_queries(query_get_count_distincts_from_object)
-
+            dict_count_distincts=self.execute_queries(query_get_count_distincts_from_object)
+        
         except Exception as err:
-            # raise err
-            dict_count_distincts = [{"COUNT_DISTINCT": 0}]
-            error_list.append(["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]])
+            #raise err
+            dict_count_distincts = [{'COUNT_DISTINCT': 0}]
+            error_list.append(["ERROR", str(err).split('|||')[0], str(err).split('|||')[1]])
 
+        
         return dict_count_distincts, error_list
 
-    def get_table_size(self, object: DatabaseObject) -> int:
-        """returns size of given object
+    def get_table_size(self, object : DatabaseObject) -> int:
+        """ returns size of given object 
 
         Args:
             object (DatabaseObject): table or view
@@ -339,15 +332,8 @@ class SnowflakeService:
 
         return size
 
-    def create_checksums(
-        self,
-        object: DatabaseObject,
-        column_intersections: list,
-        where_clause: str = "",
-        exclude_columns: list = [],
-        numeric_scale: int = None,
-    ) -> list[dict]:
-        """creates checksums for given object in compliance with given conditions
+    def create_checksums(self, object : DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns:list=[], numeric_scale: int = None) -> List[Dict]:
+        """ creates checksums for given object in compliance with given conditions
 
         Args:
             object (DatabaseObject): table or view
@@ -376,67 +362,66 @@ class SnowflakeService:
             count_nulls += f", SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END) AS COUNTNULLS_{column}"
 
             if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
+
                 if numeric_scale:
-                    aggregates += (
-                        f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS sum_{column}"
-                    )
+                    aggregates += f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS sum_{column}"
                 else:
                     aggregates += f", CAST(SUM({column}) AS DECIMAL(38)) AS sum_{column}"
 
             elif (
-                column_datatype.lower() in self.snowflake_datatype_mapping["string"]
-                or column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]
+                column_datatype.lower()  in self.snowflake_datatype_mapping["string"] 
+                or column_datatype.lower()  in self.snowflake_datatype_mapping["date_and_time"]
             ):
+
                 aggregates += f", COUNT(DISTINCT LOWER({column})) AS countdistinct_{column}"
 
             elif column_datatype.lower() in self.snowflake_datatype_mapping["binary"]:
+
                 aggregates += f", COUNT(DISTINCT LOWER(TRY_TO_NUMBER({column}::VARCHAR))) AS countdistinct_{column}"
 
             elif column_datatype.lower() in self.snowflake_datatype_mapping["boolean"]:
+
                 aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = false) :: VARCHAR AS aggregateboolean_{column}"
 
-            # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
+            #else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
-        query_checksums = (
-            f"SELECT {aggregates[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
-        )
+        query_checksums = f"SELECT {aggregates[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
 
-        query_countnulls = (
-            f"SELECT {count_nulls[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
-        )
+        query_countnulls = f"SELECT {count_nulls[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
 
         error_list = []
-        test_list = []
-        aggregation_results = {}
+        test_list=[]
+        aggregation_results={}
 
         try:
-            checksums_results = self.execute_queries([query_checksums, query_countnulls])
+            checksums_results = self.execute_queries([query_checksums,query_countnulls])
 
-            aggregation_results = checksums_results[0][0]
+            aggregation_results=checksums_results[0][0]
 
-            countnulls_results = checksums_results[1][0]
+            countnulls_results=checksums_results[1][0]
 
-            for i in range(0, len(aggregation_results)):
+            for i in range(0,len(aggregation_results)):
+                
                 if list(aggregation_results.values())[i] is None:
                     agg_result = 0
                 else:
                     agg_result = list(aggregation_results.values())[i]
-
+                
                 if list(countnulls_results.values())[i] is None:
                     cnt_result = 0
                 else:
                     cnt_result = list(countnulls_results.values())[i]
 
-                test_list.append(
-                    [[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i], agg_result, cnt_result]
-                )
+
+                test_list.append([[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i],agg_result,cnt_result])
 
         except Exception as err:
-            error_list.append(["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]])
+            error_list.append(["ERROR", str(err).split('|||')[0], str(err).split('|||')[1]])
 
-        checksums = dict(zip([item.split("_", 1)[1] for item in aggregation_results.keys()], test_list))
-        checksums["TESTATM_ERRORS"] = error_list
 
+        checksums = dict(zip([item.split("_", 1)[1] for item in aggregation_results.keys()] , test_list))
+        checksums['TESTATM_ERRORS'] = error_list
+        
         return checksums
 
     def create_pandas_df_from_group_by(
@@ -449,8 +434,8 @@ class SnowflakeService:
         only_numeric: bool,
         where_clause: str,
         exclude_columns: list,
-        numeric_scale: int = None,
-    ) -> list[dict]:
+        numeric_scale: int = None
+    ) -> List[Dict]:
         """execution of multiple aggregations at once
 
         Args:
@@ -465,24 +450,16 @@ class SnowflakeService:
             numeric_scale (int, optional): number of decimal places for aggregations. Defaults to None.
 
         Returns:
-            List[Dict]: list of pandas dataframes with results from aggregations, used sql queries
+            List[Dict]: list of pandas dataframes with results from aggregations, used sql queries 
         """
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
         if group_by_aggregation_columns == ["all"]:
-            aggregation_columns = [
-                f"{column.upper()}"
-                for column in column_intersections
-                if (column not in group_by_columns and column not in exclude_columns)
-            ]
+            aggregation_columns= [f"{column.upper()}" for column in column_intersections if (column not in group_by_columns and column not in exclude_columns)]
         else:
-            aggregation_columns = [
-                f"{column.upper()}"
-                for column in column_intersections
-                if (column in group_by_aggregation_columns and column not in exclude_columns)
-            ]
+            aggregation_columns= [f"{column.upper()}" for column in column_intersections if (column in group_by_aggregation_columns and column not in exclude_columns)]
 
         group_by_query_columns_string = " "
         grouping_columns_final = []
@@ -496,15 +473,16 @@ class SnowflakeService:
 
             group_by_query_columns_string = group_by_query_columns_string[:-1]
 
-            dict_colummns_datatype = self.get_data_types_from_object(object, aggregation_columns)
+            dict_colummns_datatype=self.get_data_types_from_object(object, aggregation_columns)
 
             aggregates = ""
             aggregates_min = ""
 
             for column in aggregation_columns:
-                column_datatype = next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
 
-                if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
+                column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
+
+                if column_datatype.lower() in  self.snowflake_datatype_mapping["numeric"]:
                     if numeric_scale:
                         aggregates_min += f", CAST(ROUND(MIN({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MIN_{column}, CAST(ROUND(max({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MAX_{column}"
                         aggregates += f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS SUM_{column}"
@@ -512,19 +490,19 @@ class SnowflakeService:
                         aggregates_min += f", MIN({column}) AS MIN_{column}, MAX({column}) AS MAX_{column}"
                         aggregates += f", SUM({column}) AS SUM_{column}"
 
-                elif not only_numeric and (
-                    column_datatype.lower() in self.snowflake_datatype_mapping["string"]
-                    or column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]
-                ):
+                elif not only_numeric and (column_datatype.lower() in  self.snowflake_datatype_mapping["string"] or column_datatype.lower() in  self.snowflake_datatype_mapping["date_and_time"]):
+
                     aggregates += f", COUNT(DISTINCT LOWER({column})) AS COUNTDISTINCT_{column}"
 
-                elif not only_numeric and column_datatype.lower() in self.snowflake_datatype_mapping["binary"]:
+                elif not only_numeric and column_datatype.lower() in  self.snowflake_datatype_mapping["binary"]:
+
                     aggregates += f", COUNT(DISTINCT LOWER(TRY_TO_NUMBER({column}::VARCHAR))) AS COUNTDISTINCT_{column}"
 
-                elif not only_numeric and column_datatype.lower() in self.snowflake_datatype_mapping["boolean"]:
-                    aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = false) :: VARCHAR AS AGGREGATEBOOLEAN_{column}"
+                elif not only_numeric and column_datatype.lower() in  self.snowflake_datatype_mapping["boolean"]:
 
-                # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
+                    aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = false) :: VARCHAR AS AGGREGATEBOOLEAN_{column}"
+                
+                #else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
             # CASE 1: min_max
             if group_by_aggregation_type == "only_min_max":
@@ -537,44 +515,35 @@ class SnowflakeService:
             # CASE 3: sum, count_distinct, aggregate_boolean, min_max
             elif group_by_aggregation_type == "various_and_min_max":
                 group_by_query_aggregation_string = f"{aggregates_min[1:]}{aggregates}"
-
+        
             query_group_by_aggregation = f"SELECT {group_by_query_columns_string}, COUNT(*) AS COUNT_OF_GROUP_BY_VALUE, {group_by_query_aggregation_string} FROM {object.database}.{object.schema}.{object.name} {where_clause} GROUP BY {group_by_query_columns_string} ORDER BY {group_by_query_columns_string};"
 
-            group_by_aggregation_pdf = self.execute_queries(query_group_by_aggregation, True)
+            group_by_aggregation_pdf = self.execute_queries(query_group_by_aggregation,True)
         except Exception as err:
             group_by_aggregation_pdf = pd.DataFrame()
             group_by_aggregation_pdf["TESTATM_ERROR"] = [1]
             if not grouping_columns_final:
                 error_dict = {
                     "QUERY": "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table",
-                    "ERROR": "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table",
+                    "ERROR":  "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table"
                 }
                 group_by_query_aggregation_string = ""
-            elif "|||" in str(err):
-                error_dict = {"QUERY": str(err).split("|||")[0], "ERROR": str(err).split("|||")[1]}
+            elif '|||' in str(err):
+                error_dict = {
+                    "QUERY": str(err).split('|||')[0],
+                    "ERROR":  str(err).split('|||')[1]
+                }
             else:
                 error_dict = {
                     "QUERY": "NO Query generated. Please check if the configurated Grouping Columns exist in the Table",
-                    "ERROR": str(err),
+                    "ERROR":  str(err)
                 }
                 group_by_query_aggregation_string = ""
 
-        return (
-            group_by_aggregation_pdf,
-            group_by_query_aggregation_string,
-            group_by_query_columns_string,
-            grouping_columns_final,
-            error_dict,
-        )
+        return group_by_aggregation_pdf, group_by_query_aggregation_string, group_by_query_columns_string, grouping_columns_final, error_dict
 
-    def create_pandas_df(
-        self,
-        object: DatabaseObject,
-        intersection_columns_trgt_src: list,
-        where_clause: str = "",
-        exclude_columns: list = [],
-    ) -> pd.DataFrame:
-        """creates pandas dataframes with all data from given object in given columns
+    def create_pandas_df(self, object : DatabaseObject, intersection_columns_trgt_src: list, where_clause:str="", exclude_columns:list=[]) -> pd.DataFrame:
+        """ creates pandas dataframes with all data from given object in given columns
 
         Args:
             object (DatabaseObject): table or view
@@ -587,26 +556,16 @@ class SnowflakeService:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        intersection_columns_trgt_src_ = ", ".join(list(set(intersection_columns_trgt_src) - set(exclude_columns)))
+        intersection_columns_trgt_src_ = ', '.join(list(set(intersection_columns_trgt_src) - set(exclude_columns)))
 
         df_query = f"SELECT {intersection_columns_trgt_src_} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
-
-        src_pdf = self.execute_queries(df_query, True)
+            
+        src_pdf = self.execute_queries(df_query,True)
 
         return src_pdf
 
-    def create_pandas_df_from_sample(
-        self,
-        object: DatabaseObject,
-        column_intersections: list,
-        key_columns: list,
-        where_clause: str = "",
-        exclude_columns: list = [],
-        key_filters: dict = {},
-        dedicated_columns: list = [],
-        sample_count: int = 10,
-        numeric_scale: int = None,
-    ) -> list[dict]:
+    def create_pandas_df_from_sample(self, object: DatabaseObject, column_intersections: list, key_columns: list, where_clause:str="", exclude_columns:list=[], key_filters: dict={}, dedicated_columns: list=[], sample_count :int=10, numeric_scale: int = None) -> List[Dict]:
+ 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
@@ -620,37 +579,34 @@ class SnowflakeService:
         dedicated_intersection.sort()
 
         if not where_clause:
-            where_clause = "WHERE 1=1 "
+            where_clause= 'WHERE 1=1 '
 
         if dedicated_intersection != []:
             is_dedicated = True
 
-            dict_colummns_datatype = self.get_data_types_from_object(object, dedicated_intersection)
+            dict_colummns_datatype=self.get_data_types_from_object(object, dedicated_intersection)
 
         else:
             is_dedicated = False
 
-            dict_colummns_datatype = self.get_data_types_from_object(object, column_intersections)
+            dict_colummns_datatype=self.get_data_types_from_object(object, column_intersections)
 
+       
         if key_intersection != [] and is_dedicated:
             keys = str(key_intersection)[1:-1].replace("'", "")
-            column_clause, numeric_columns, used_columns = self._get_column_clause(
-                dedicated_intersection, dict_colummns_datatype, numeric_scale, key_columns
-            )
+            column_clause, numeric_columns, used_columns = self._get_column_clause(dedicated_intersection, dict_colummns_datatype, numeric_scale, key_columns)
             if (key_filters != {}) & (filter_intersection != []):
                 values = list(key_filters.values())
                 if values[0] != []:
-                    in_clause = self._get_in_clause(key_filters, numeric_columns, numeric_scale)
+                    in_clause = self._get_in_clause(key_filters, numeric_columns, numeric_scale) 
                 else:
                     in_clause = ""
             else:
-                in_clause = ""
+                in_clause = "" 
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause}{in_clause} ORDER BY {keys};"
         elif key_intersection != [] and not is_dedicated:
             keys = str(key_intersection)[1:-1].replace("'", "")
-            column_clause, numeric_columns, used_columns = self._get_column_clause(
-                column_intersections, dict_colummns_datatype, numeric_scale, key_columns
-            )
+            column_clause, numeric_columns, used_columns = self._get_column_clause(column_intersections, dict_colummns_datatype, numeric_scale, key_columns)
             if (key_filters != {}) & (filter_intersection != []):
                 values = list(key_filters.values())
                 if values[0] != []:
@@ -661,11 +617,9 @@ class SnowflakeService:
                 in_clause = ""
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause}{in_clause} ORDER BY {keys};"
         else:
-            column_intersections = list(set(column_intersections) - set(exclude_columns))
+            column_intersections = list(set(column_intersections)  - set(exclude_columns))
             column_intersections.sort()
-            column_clause, numeric_columns, used_columns = self._get_column_clause(
-                column_intersections, dict_colummns_datatype, numeric_scale, key_columns
-            )
+            column_clause, numeric_columns, used_columns = self._get_column_clause(column_intersections, dict_colummns_datatype, numeric_scale, key_columns)
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause};"
 
         error_dict = {}
@@ -681,21 +635,26 @@ class SnowflakeService:
         except Exception as err:
             sample_pdf = pd.DataFrame()
             sample_pdf["TESTATM_ERROR"] = [1]
-            if "|||" in str(err):
-                error_dict = {"QUERY": str(err).split("|||")[0], "ERROR": str(err).split("|||")[1]}
+            if '|||' in str(err):
+                error_dict = {
+                    "QUERY": str(err).split('|||')[0],
+                    "ERROR":  str(err).split('|||')[1]
+                }
             else:
-                error_dict = {"QUERY": "No SQL Error", "ERROR": str(err)}
+                error_dict = {
+                    "QUERY": 'No SQL Error',
+                    "ERROR":  str(err)
+                }
 
         return_list = []
         return_list.append(sample_pdf)
         return_list.append(error_dict)
 
-        return return_list, key_dict, used_columns, sample_query
 
-    def execute_queries(
-        self, query: str | list[str], return_as_pdf: bool = False, return_query_ids: bool = False
-    ) -> list[dict] | list[list[dict]]:
-        """actual execution of defined queries
+        return return_list , key_dict, used_columns, sample_query
+
+    def execute_queries(self, query: Union[str, List[str]],return_as_pdf:bool=False, return_query_ids:bool=False)  -> Union[List[Dict], List[List[Dict]]]:
+        """ actual execution of defined queries
 
         Args:
             query (Union[str, List[str]]): queries to be executed
@@ -711,23 +670,23 @@ class SnowflakeService:
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-
+        
         if query:
-            query_list: list[str] = query if isinstance(query, list) else [query]
+            query_list: List[str] = query if isinstance(query, list) else [query]
         else:
-            logger.error("Query defined as null - please check input for execute_queries function.")
+            logger.error('Query defined as null - please check input for execute_queries function.')
 
-        cursor = self.snowflake_connection.get_connection_object().cursor(snowflake.connector.DictCursor)
+        cursor = self.snowflake_connection.cursor(snowflake.connector.DictCursor)
 
         results = []
-        query_ids = []
+        query_ids=[]
 
         for single_query in query_list:
-            try:
+            try:            
                 query_result = cursor.execute(single_query).fetchall()
                 if return_as_pdf:
                     query_result = pd.DataFrame(query_result)
-
+                
                 results.append(query_result)
                 query_ids.append(cursor.sfqid)
 
@@ -740,7 +699,7 @@ class SnowflakeService:
         else:
             return results[0] if not isinstance(query, list) else results
 
-    def execute_statement(self, statement: str | list[str]) -> None:
+    def execute_statement(self, statement: Union[str, List[str]]) -> None:
         """
             Executes simple statement against snowflake
             Schema and Database settings must be set beforehand
@@ -749,18 +708,23 @@ class SnowflakeService:
         """
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-
-        statement_list: list[str] = statement if isinstance(statement, list) else [statement]
+            
+        statement_list: List[str] = (
+            statement if isinstance(statement, list) else [statement]
+        )
 
         try:
             for single_statement in statement_list:
-                stripped_statement = single_statement.strip()
-                _ = self.snowflake_connection.get_connection_object().execute_string(stripped_statement)
+                stripped_statement = (
+                    single_statement.strip()
+                )  
+                _ = self.snowflake_connection.execute_string(stripped_statement)
 
         except Exception as err:
             raise Exception(self._get_error_message(err, single_statement)) from err
-
+        
     def upload_to_stage(self, stage_name: str, folder_path: str, file_name: str, is_temporary: bool):
+        
         file_path = PurePath(folder_path).joinpath(PurePath(file_name))
 
         if is_temporary:
@@ -770,70 +734,48 @@ class SnowflakeService:
 
         put_query = rf"PUT 'file://{file_path}' @{stage_name};"
 
-        put_query = put_query.replace("\\", "\\\\")
+        put_query = put_query.replace("\\","\\\\")
 
         self.execute_statement(create_query)
 
         self.execute_statement(put_query)
 
-    def insert_json_results(
-        self,
-        run_guid: str,
-        pipeline_name: str,
-        pipeline_id: str,
-        start_time_utc: str,
-        result_table: str,
-        stage_name: str,
-    ) -> None:
+    def insert_json_results(self, run_guid: str, pipeline_name: str, pipeline_id: str, start_time_utc: str, result_table: str, stage_name: str ) -> None: 
         """
-        copy into - result table for json results
+            copy into - result table for json results
         """
-        result_database = result_table.split(".")[0]
-        meta_data_schema = result_table.split(".")[1]
+        result_database = result_table.split('.')[0]
+        meta_data_schema = result_table.split('.')[1]
 
         statement = f"COPY INTO {result_table} (RUN_GUID, PIPELINE_NAME, PIPELINE_ID, START_TIME_UTC, RESULT, CREATION_TIME_UTC) FROM (SELECT '{run_guid}', '{pipeline_name}', '{pipeline_id}', '{start_time_utc}', $1, SYSDATE() from @{stage_name} (file_format => {result_database}.{meta_data_schema}.ff_json ));"
 
         self.execute_statement(statement)
 
-    def insert_json_results_live(
-        self,
-        run_guid: str,
-        pipeline_name: str,
-        pipeline_id: str,
-        result_table: str,
-        stage_name: str,
-        source_system: str,
-        target_system: str,
-        database: str,
-        schema: str,
-        object: str,
-    ) -> None:
+    def insert_json_results_live(self, run_guid: str, pipeline_name: str, pipeline_id: str,  result_table: str, stage_name: str , source_system: str, target_system:str, database:str, schema:str, object:str) -> None: 
         """
-        copy into - result table for json results live
+            copy into - result table for json results live
         """
-        result_database = result_table.split(".")[0]
-        meta_data_schema = result_table.split(".")[1]
+        result_database = result_table.split('.')[0]
+        meta_data_schema = result_table.split('.')[1]
 
         statement = f"COPY INTO {result_table} (RUN_GUID, PIPELINE_NAME, PIPELINE_ID, SOURCE_SYSTEM, TARGET_SYSTEM, DATABASE_NAME, SCHEMA_NAME, OBJECT_NAME  ,RESULT, CREATION_TS) FROM (SELECT '{run_guid}', '{pipeline_name}', '{pipeline_id}', '{source_system}', '{target_system}', '{database}', '{schema}', '{object}', $1, SYSDATE() from @{stage_name} (file_format => {result_database}.{meta_data_schema}.ff_json ));"
 
         self.execute_statement(statement)
 
-    def insert_highlevel_results(
-        self, results: dict, run_guid: str, pipeline_name: str, pipeline_id: str, result_table_highlevel: str
-    ) -> None:
+    def insert_highlevel_results(self, results: dict, run_guid:str, pipeline_name: str, pipeline_id: str, result_table_highlevel: str) -> None: 
         """
-        insert into - highlevel results per "pipeline run" / "ics data validation execution"
+            insert into - highlevel results per "pipeline run" / "ics data validation execution"
         """
-        TESTSET_ = ", ".join(results["TESTSET"])
+        TESTSET_ = ', '.join(results['TESTSET'])
 
-        OBJECTS_TO_COMPARE_SRC_ = ", ".join(results["OBJECTS_TO_COMPARE_SRC"])
+        OBJECTS_TO_COMPARE_SRC_ = ', '.join(results['OBJECTS_TO_COMPARE_SRC'])
 
-        OBJECTS_TO_COMPARE_TRGT_ = ", ".join(results["OBJECTS_TO_COMPARE_TRGT"])
+        OBJECTS_TO_COMPARE_TRGT_ = ', '.join(results['OBJECTS_TO_COMPARE_TRGT'])
 
-        SRC_MINUS_TRGT_ = ", ".join(results["SRC_MINUS_TRGT"])
-
-        TRGT_MINUS_SRC_ = ", ".join(results["TRGT_MINUS_SRC"])
+        SRC_MINUS_TRGT_ = ', '.join(results['SRC_MINUS_TRGT'])
 
+        TRGT_MINUS_SRC_ = ', '.join(results['TRGT_MINUS_SRC'])
+        
         insert_statement = f"INSERT INTO {result_table_highlevel} ( \
                                                                         RUN_GUID, \
                                                                         PIPELINE_NAME,  \
@@ -877,13 +819,13 @@ class SnowflakeService:
                                                                         '{results['NUMBER_OF_OBJECTS_TO_COMPARE']}',  \
                                                                         '{SRC_MINUS_TRGT_}',  \
                                                                         '{TRGT_MINUS_SRC_}', \
-                                                                        SYSDATE())"
-
+                                                                        SYSDATE())"  
+                                                                        
         self.execute_statement(insert_statement)
 
-    def insert_objectlevel_results(self, result_table: dict, result_table_objectlevel: str, run_guid: str) -> None:
+    def insert_objectlevel_results(self, result_table: dict, result_table_objectlevel: str, run_guid:str) -> None:
         """
-        insert into - detailed results per object
+            insert into - detailed results per object
         """
         insert_statement = f"INSERT INTO {result_table_objectlevel} ( \
                                                                         RUN_GUID, \
@@ -1012,14 +954,15 @@ class SnowflakeService:
                                             FROM {result_table} RESULTS \
                                             CROSS JOIN LATERAL FLATTEN(INPUT => RESULT:OBJECTS) F1\
                                             WHERE RUN_GUID = '{run_guid}'\
-                                ;"
+                                ;" 
 
         self.execute_statement(insert_statement)
 
-    def insert_columnlevel_results(self, result_table: str, result_table_columnlevel: str, run_guid: str) -> None:
-        """
-        insert into - detailed results per column
+
+    def insert_columnlevel_results(self, result_table: str, result_table_columnlevel: str, run_guid:str) -> None: 
         """
+            insert into - detailed results per column
+        """   
         insert_statement = f"INSERT INTO {result_table_columnlevel} ( \
                                                                 RUN_GUID,\
                                                                 PIPELINE_ID,\
@@ -1096,5 +1039,5 @@ class SnowflakeService:
                                                             CROSS JOIN LATERAL FLATTEN(INPUT => RESULT:OBJECTS) F1\
                                                             CROSS JOIN LATERAL FLATTEN(INPUT => F1.VALUE:COLUMNS) F2\
                                                             WHERE RUN_GUID = '{run_guid}';"
-
-        self.execute_statement(insert_statement)
+        
+        self.execute_statement(insert_statement)

icsDataValidation-1.0.365.dist-info/METADATA

@@ -0,0 +1,20 @@
+Metadata-Version: 2.1
+Name: icsDataValidation
+Version: 1.0.365
+Summary: ics data validation
+Home-page: https://initions.com/
+Author: initions
+Author-email: ICSMC_EXT_PYPIORG@accenture.com
+License: MIT
+Requires-Dist: snowflake-connector-python[pandas] (==3.7.1)
+Requires-Dist: python-dotenv (==0.20.0)
+Requires-Dist: pyodbc
+Requires-Dist: pyexasol (==0.24.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: azure-storage-blob (==12.13.1)
+Requires-Dist: teradatasql (==17.20.0.10)
+Requires-Dist: boto3 (==1.26.154)
+Requires-Dist: oracledb (==2.5.0)
+Requires-Dist: databricks-sql-connector (==3.0.1)
+Requires-Dist: databricks-sdk (==0.29.0)
+Requires-Dist: numpy (==1.26.3)

{icsDataValidation-1.0.361.dist-info → icsDataValidation-1.0.365.dist-info}/RECORD

@@ -1,3 +1,5 @@
+examples/ics_data_validation.py,sha256=vyBAnU8yQGKGH33ZxrvaZpY-kt1iQ3h53kzkKG0Y7gI,139
+examples/manual_execution_params.template.py,sha256=g3LAah1zEXJtozAZFpkxCm-JCWXSQY3R2SG-8YcPV9c,2038
 icsDataValidation/configuration.py,sha256=HOFjmC8_e2nvoItndMtJQQA1MR5aCgZGeF1AwY_FvjE,477
 icsDataValidation/main.py,sha256=nmbFM8Epf4-Nhd9ArH31wT7Yx0MSjIHxX93zPke1ArA,12498
 icsDataValidation/connection_setups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -5,7 +7,7 @@ icsDataValidation/connection_setups/azure_connection_setup.py,sha256=gvTyctG63ol
 icsDataValidation/connection_setups/databricks_connection_setup.py,sha256=dNEBum-8R-TUW2SCEk3CaNtCr_gLFvn456KBlENpgJU,1220
 icsDataValidation/connection_setups/exasol_connection_setup.py,sha256=RfCUsL6G-NaOW-qNK-3SfHcljbRaKD6fDIHXkNQhClk,590
 icsDataValidation/connection_setups/oracle_connection_setup.py,sha256=D-4ucC1ChE4HYm93ECIEg_yBOrn1NkknxFBgFRGFmWs,978
-icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=IgEhni4Q0oYGh2QzptpyfEUvUt3cVO28jNSGg11cxyI,1778
+icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=JDTdIM0bQ0_IV0HnCct90RC6Mq4fS1sfh9IJc-YYlMo,804
 icsDataValidation/connection_setups/teradata_connection_setup.py,sha256=fIpuxz-FTqFK2vSMSuokqU9sdJkaJ4UP5piY_zIbj5k,624
 icsDataValidation/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/core/database_objects.py,sha256=2oaDaVQajSYI_HJjJy1pmc6FsoK_wMfwgu6ZgEcFvow,523
@@ -15,7 +17,7 @@ icsDataValidation/input_parameters/testing_tool_params.py,sha256=6LkqEaH3vaeCn6d
 icsDataValidation/output_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/output_parameters/result_params.py,sha256=5Mk9L9zWaxUqcKwLZQ539lVUp0b0s-YUmSA3PBgbqfs,2833
 icsDataValidation/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/services/comparison_service.py,sha256=x8RjZs2bgYDHFueq5ysNADY1cC6rYBf6eDyCsF-_w84,42705
+icsDataValidation/services/comparison_service.py,sha256=WhlkVY9I28jBsksFF-cF-OwNIIwigyuuiLJTTAF8HvY,44244
 icsDataValidation/services/initialization_service.py,sha256=AHbJrq_LjMPFoeOJC2pi2ZZ1xkL8njSZn38psc3do60,6687
 icsDataValidation/services/result_service.py,sha256=edD6aejIi5P7qDNHKnN46KrN5tfzwqnw5TB35SvFAWU,28396
 icsDataValidation/services/system_service.py,sha256=GSkSPNG5PlLWchwlYM5H-1FMtuCNwpXcyZZOUB_0stU,3228
@@ -26,7 +28,7 @@ icsDataValidation/services/database_services/databricks_hive_metastore_service.p
 icsDataValidation/services/database_services/databricks_unity_catalog_service.py,sha256=INA8rd3KW_jAplNagGa9tEON3dyOufcIAPOOdmc0Mrc,70259
 icsDataValidation/services/database_services/exasol_service.py,sha256=7LYnRScO3DxBmuSN0HmTgsFc2el-Ii3A9jgGsXSJVU8,11074
 icsDataValidation/services/database_services/oracle_service.py,sha256=60unwWlHm520ioFmz0y2K8ApwZrruf9iB0ojjQx0IWc,31523
-icsDataValidation/services/database_services/snowflake_service.py,sha256=EYOZjkjeh0CMGApef-LWoXP4JeJzhAG_qUCqpwOQ9ek,61021
+icsDataValidation/services/database_services/snowflake_service.py,sha256=UWmjQZN4oX2ctH6uhE2oklXGHo66SK5UnQbFYFhzDuc,60630
 icsDataValidation/services/database_services/teradata_service.py,sha256=Rf0xzcZGEbooq3r2Rfe2fCahTm2Xw4uznQa8vyWoyqM,40169
 icsDataValidation/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/utils/file_util.py,sha256=ZTMB1sTnIIdffg9tEJRCFQQ5SG8Fksc5ie1PM4gHXG4,3432
@@ -34,7 +36,7 @@ icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI
 icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
 icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
 icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
-icsDataValidation-1.0.361.dist-info/METADATA,sha256=m3_7gnPsag7iS3Kg02_JjMV7azhFZqR4H6nUK70dlnw,24605
-icsDataValidation-1.0.361.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-icsDataValidation-1.0.361.dist-info/top_level.txt,sha256=BqWUGJb4J7ZybpDMeuGHxEHGHwXXJEIURd9pBybHzTM,18
-icsDataValidation-1.0.361.dist-info/RECORD,,
+icsDataValidation-1.0.365.dist-info/METADATA,sha256=R_1JkIr8O6xWtIIh1E6u2aOypk7Qyti22Pl5ls2BdYw,720
+icsDataValidation-1.0.365.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+icsDataValidation-1.0.365.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
+icsDataValidation-1.0.365.dist-info/RECORD,,

{icsDataValidation-1.0.361.dist-info → icsDataValidation-1.0.365.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: bdist_wheel (0.37.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

icsDataValidation-1.0.365.dist-info/top_level.txt

@@ -0,0 +1,4 @@
+azure-pipelines
+examples
+icsDataValidation
+resources

icsDataValidation-1.0.361.dist-info/METADATA

@@ -1,21 +0,0 @@
-Metadata-Version: 2.2
-Name: icsDataValidation
-Version: 1.0.361
-Summary: Add your description here
-Home-page: https://initions.com/
-Author: initions
-Author-email: ICSMC_EXT_PYPIORG@accenture.com
-License: MIT
-Requires-Python: >=3.11
-Requires-Dist: azure-storage-blob==12.13.1
-Requires-Dist: boto3==1.26.154
-Requires-Dist: cloe-util-snowflake-connector==1.0.5
-Requires-Dist: databricks-sdk==0.29.0
-Requires-Dist: databricks-sql-connector==3.0.1
-Requires-Dist: numpy==1.26.3
-Requires-Dist: oracledb==2.5.0
-Requires-Dist: pandas==2.2.2
-Requires-Dist: pyexasol==0.24.0
-Requires-Dist: pyodbc
-Requires-Dist: python-dotenv>=1.0.1
-Requires-Dist: teradatasql==17.20.0.10

icsDataValidation-1.0.361.dist-info/top_level.txt

@@ -1 +0,0 @@
-icsDataValidation