icsDataValidation 1.0.360__py3-none-any.whl → 1.0.361__py3-none-any.whl

icsDataValidation/connection_setups/snowflake_connection_setup.py

@@ -1,20 +1,35 @@
 import os
 
-from dotenv import load_dotenv
-from pathlib import Path
+from cloe_util_snowflake_connector.connection_parameters import ConnectionParameters, EnvVariablesInitializer
 
 #########################################################################################
 #########################################################################################
 
-def load_snowflake_credentials(system_configs:dict,system_selection:str)->dict:
 
-    snowflake_params = {
-        "account"   : system_configs[system_selection]["ACCOUNT"],
-        "user"      : system_configs[system_selection]["USER"],
-        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"]),
-        "warehouse" : system_configs[system_selection]["WAREHOUSE"],
-        "role"      : system_configs[system_selection]["ROLE"],
-        "database"  : system_configs[system_selection]["DATABASE"]
-    }
+def load_snowflake_credentials(system_configs: dict, system_selection: str) -> ConnectionParameters:
+    snowflake_params = EnvVariablesInitializer(
+        user=system_configs[system_selection]["USER"],
+        account=system_configs[system_selection]["ACCOUNT"],
+        warehouse=system_configs[system_selection]["WAREHOUSE"],
+        database=system_configs[system_selection]["DATABASE"],
+        role=system_configs[system_selection]["ROLE"],
+        password=os.getenv(system_configs[system_selection]["PASSWORD_NAME"])
+        if "PASSWORD_NAME" in system_configs[system_selection]
+        else None,
+        private_key=os.getenv(system_configs[system_selection]["PRIVATE_KEY_NAME"])
+        if "PRIVATE_KEY_NAME" in system_configs[system_selection]
+        else None,
+        private_key_passphrase=os.getenv(system_configs[system_selection]["PRIVATE_KEY_PASSPHRASE_NAME"])
+        if "PRIVATE_KEY_PASSPHRASE_NAME" in system_configs[system_selection]
+        else None,
+        private_key_file=os.getenv(system_configs[system_selection]["PRIVATE_KEY_FILE_PATH"])
+        if "PRIVATE_KEY_FILE_PATH" in system_configs[system_selection]
+        else None,
+        private_key_file_pwd=os.getenv(system_configs[system_selection]["PRIVATE_KEY_FILE_PASSWORD"])
+        if "PRIVATE_KEY_FILE_PASSWORD" in system_configs[system_selection]
+        else None,
+    )
 
-    return snowflake_params
+    connection_params = ConnectionParameters(**snowflake_params.model_dump())
+
+    return connection_params

icsDataValidation/services/database_services/snowflake_service.py

@@ -1,34 +1,33 @@
-
-import snowflake.connector
-import pandas as pd
 import logging
-
-from typing import Union, List, Dict 
 from pathlib import PurePath
 
-from icsDataValidation.utils.logger_util import configure_dev_ops_logger
+import pandas as pd
+import snowflake.connector
+from cloe_util_snowflake_connector import connection_parameters, snowflake_interface
+
 from icsDataValidation.core.database_objects import DatabaseObject
+from icsDataValidation.utils.logger_util import configure_dev_ops_logger
 
 #########################################################################################
 #########################################################################################
 
 # Configure Dev Ops Logger
 
-logger = logging.getLogger('Snowflake_Service')
+logger = logging.getLogger("Snowflake_Service")
 logger.setLevel(logging.INFO)
 configure_dev_ops_logger(logger)
 
-class SnowflakeService(object):
 
-    def __init__(self, connection_params: dict):
-        self.connection_params =connection_params
+class SnowflakeService:
+    def __init__(self, connection_params: connection_parameters.ConnectionParameters):
+        self.connection_params = connection_params
         self.snowflake_connection = None
         self.snowflake_datatype_mapping = {
-                                            "string": ['text'],
-                                            "numeric": ['number', 'float'],
-                                            "date_and_time" : ['date', 'time', 'timestamp_ntz', 'timestamp_tz', 'timestamp_ltz'],
-                                            "binary" : ['binary'],
-                                            "boolean" : ['boolean']
+            "string": ["text"],
+            "numeric": ["number", "float"],
+            "date_and_time": ["date", "time", "timestamp_ntz", "timestamp_tz", "timestamp_ltz"],
+            "binary": ["binary"],
+            "boolean": ["boolean"],
         }
 
     def __enter__(self):
@@ -43,7 +42,7 @@ class SnowflakeService(object):
             self.snowflake_connection.close()
 
     def _connect_to_snowflake(self):
-        self.snowflake_connection = snowflake.connector.connect(**self.connection_params)
+        self.snowflake_connection = snowflake_interface.SnowflakeInterface(self.connection_params)
         return self.snowflake_connection
 
     @staticmethod
@@ -62,8 +61,8 @@ class SnowflakeService(object):
         return f"Snowflake ERROR: {message}\nFailed statement:\n{statement}"
 
     @staticmethod
-    def _get_in_clause(key_filters:list, numeric_columns:list, numeric_scale:int) -> str:
-        """ generates in_clause from list ready to expand the where clause, numeric values are rounded
+    def _get_in_clause(key_filters: list, numeric_columns: list, numeric_scale: int) -> str:
+        """generates in_clause from list ready to expand the where clause, numeric values are rounded
 
         Args:
             key_filters (list): list of given expected values
@@ -72,26 +71,26 @@ class SnowflakeService(object):
 
         Returns:
             str: in clause as string
-        """        
-        values = list(key_filters.values())  
+        """
+        values = list(key_filters.values())
         in_clause_values = "('"
         for j in range(len(values[0])):
             for value in values:
                 in_clause_values += str(value[j]) + "','"
             in_clause_values = in_clause_values[:-2] + "),('"
-        in_clause_values = in_clause_values[:-3] + ')'
+        in_clause_values = in_clause_values[:-3] + ")"
 
-        in_clause_cols = f" AND ((" 
+        in_clause_cols = " AND (("
         for key in key_filters.keys():
             if key in numeric_columns:
                 in_clause_cols += f"""ROUND({key.replace("'", "")},2)""" + ","
             else:
                 in_clause_cols += key.replace("'", "") + ","
         in_clause_cols = in_clause_cols[:-1] + ")"
-        in_clause = in_clause_cols + " in ("  + in_clause_values + ")"                                  
+        in_clause = in_clause_cols + " in (" + in_clause_values + ")"
         return in_clause
-    
-    def _get_column_clause(self, column_list: list, columns_datatype: list,  numeric_scale, key_columns) ->dict : 
+
+    def _get_column_clause(self, column_list: list, columns_datatype: list, numeric_scale, key_columns) -> dict:
         """
         Turns list of desired columns into a sql compatible string.
         Columns with a date or time data type are omitted.
@@ -104,23 +103,25 @@ class SnowflakeService(object):
 
         Returns:
             dict: _description_
-        """        
-        column_intersecions_new = []           
+        """
+        column_intersecions_new = []
         used_columns = []
         numeric_columns = []
         for column in column_list:
-            column_datatype=next(x for x in columns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
+            column_datatype = next(x for x in columns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
 
-            if column in  key_columns or not (column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]):
-                if column_datatype.lower() in  self.snowflake_datatype_mapping["numeric"]:
+            if column in key_columns or column_datatype.lower() not in self.snowflake_datatype_mapping["date_and_time"]:
+                if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
                     if numeric_scale:
-                        column_intersecions_new.append(f"CAST(ROUND({column}, {numeric_scale}) as decimal(38,{numeric_scale})) as {column}")
+                        column_intersecions_new.append(
+                            f"CAST(ROUND({column}, {numeric_scale}) as decimal(38,{numeric_scale})) as {column}"
+                        )
                     else:
                         column_intersecions_new.append(f"{column} as {column}")
                     used_columns.append(column)
                     numeric_columns.append(column)
-                elif column_datatype.lower() in  self.snowflake_datatype_mapping["string"]:
-                    column_intersecions_new.append(f'{column} AS {column}')
+                elif column_datatype.lower() in self.snowflake_datatype_mapping["string"]:
+                    column_intersecions_new.append(f"{column} AS {column}")
                     used_columns.append(column)
                 else:
                     column_intersecions_new.append(column)
@@ -130,44 +131,43 @@ class SnowflakeService(object):
         column_clause = str(column_intersections)[1:-1].replace("'", "")
         return column_clause, numeric_columns, used_columns
 
-    def get_database_objects(self, database: str, schema: str=None, object_type_restriction: str='include_all') -> dict:
+    def get_database_objects(
+        self, database: str, schema: str = None, object_type_restriction: str = "include_all"
+    ) -> dict:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        all_database_tables=[]
-        all_database_views=[]
+        all_database_tables = []
+        all_database_views = []
 
-        if object_type_restriction=='include_all' or object_type_restriction=='include_only_tables':
+        if object_type_restriction == "include_all" or object_type_restriction == "include_only_tables":
             if schema:
-                query_db_tables=f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE'; "
-            else:    
-                query_db_tables=f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE';"
+                query_db_tables = f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE'; "
+            else:
+                query_db_tables = f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA' AND TABLE_TYPE ='BASE TABLE';"
 
             all_database_tables = self.execute_queries(query_db_tables)
 
-
-        if object_type_restriction=='include_all' or object_type_restriction=='include_only_views':
+        if object_type_restriction == "include_all" or object_type_restriction == "include_only_views":
             if schema:
-                query_db_views=f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
-            else:    
-                query_db_views=f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
+                query_db_views = f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{schema.upper()}' AND TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
+            else:
+                query_db_views = (
+                    f"SELECT * FROM {database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA';"
+                )
 
             all_database_views = self.execute_queries(query_db_views)
-        
 
-        database_objects=[]
+        database_objects = []
         for row in all_database_tables:
-            table_identifier=f'{row["TABLE_CATALOG"]}.{row["TABLE_SCHEMA"]}.{row["TABLE_NAME"]}'
+            table_identifier = f"{row['TABLE_CATALOG']}.{row['TABLE_SCHEMA']}.{row['TABLE_NAME']}"
             database_objects.append({"object_identifier": table_identifier, "object_type": "table"})
         for row in all_database_views:
-            view_identifier=f'{row["TABLE_CATALOG"]}.{row["TABLE_SCHEMA"]}.{row["TABLE_NAME"]}'
+            view_identifier = f"{row['TABLE_CATALOG']}.{row['TABLE_SCHEMA']}.{row['TABLE_NAME']}"
             database_objects.append({"object_identifier": view_identifier, "object_type": "view"})
         return database_objects
 
-    def get_last_altered_timestamp_from_object(
-            self,
-            object: DatabaseObject
-        ) -> str:
+    def get_last_altered_timestamp_from_object(self, object: DatabaseObject) -> str:
         """queries last_altered timestamp for given object
 
         Args:
@@ -180,14 +180,14 @@ class SnowflakeService(object):
             self._connect_to_snowflake()
 
         self.execute_statement("ALTER SESSION SET TIMEZONE = 'Europe/London';")
-        
-        query_get_last_altered=f"SELECT LAST_ALTERED FROM {object.database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{object.name}' AND TABLE_SCHEMA = '{object.schema}';"
+
+        query_get_last_altered = f"SELECT LAST_ALTERED FROM {object.database}.INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{object.name}' AND TABLE_SCHEMA = '{object.schema}';"
 
         last_altered = self.execute_queries(query_get_last_altered)[0]
 
         return last_altered
 
-    def get_columns_from_object(self, object : DatabaseObject) -> list:
+    def get_columns_from_object(self, object: DatabaseObject) -> list:
         """returns all columns from given object
 
         Args:
@@ -200,30 +200,34 @@ class SnowflakeService(object):
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        if object.type =='table':
+        if object.type == "table":
             query_show_columns = f"SHOW COLUMNS IN TABLE {object.database}.{object.schema}.{object.name};"
 
-            show_columns_result, query_id, test = self.execute_queries(query_show_columns, return_as_pdf=False, return_query_ids=True)
-            
+            show_columns_result, query_id, test = self.execute_queries(
+                query_show_columns, return_as_pdf=False, return_query_ids=True
+            )
+
             query_get_columns = f"SELECT $3 AS COLUMN_NAME FROM TABLE(result_scan('{query_id}'));"
 
-        if object.type =='view':
+        if object.type == "view":
             query_show_columns = f"SHOW COLUMNS IN VIEW {object.database}.{object.schema}.{object.name};"
 
-            show_columns_result, query_id, test = self.execute_queries(query_show_columns, return_as_pdf=False, return_query_ids=True)
-            
+            show_columns_result, query_id, test = self.execute_queries(
+                query_show_columns, return_as_pdf=False, return_query_ids=True
+            )
+
             query_get_columns = f"SELECT $3 AS COLUMN_NAME FROM TABLE(result_scan('{query_id}'));"
 
         all_columns = self.execute_queries(query_get_columns)
-        columns=[]
+        columns = []
 
         for row in all_columns:
             columns.append(row["COLUMN_NAME"])
 
         return columns
 
-    def get_row_count_from_object(self, object : DatabaseObject, where_clause: str="") -> int:
-        """ gets row count from given object
+    def get_row_count_from_object(self, object: DatabaseObject, where_clause: str = "") -> int:
+        """gets row count from given object
 
         Args:
             object (DatabaseObject): table or view
@@ -234,23 +238,25 @@ class SnowflakeService(object):
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-        
-        #TODO is it more efficient to select the information_schema.table view to get the rows?
-        query_get_row_count = f"SELECT COUNT(*) AS ROW_COUNT FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+
+        # TODO is it more efficient to select the information_schema.table view to get the rows?
+        query_get_row_count = (
+            f"SELECT COUNT(*) AS ROW_COUNT FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+        )
         row_count = -1
         error_list = []
 
         try:
             row_count = self.execute_queries(query_get_row_count)[0]["ROW_COUNT"]
-        
+
         except Exception as err:
             error_list.append(str(err))
             error_list.append(query_get_row_count)
 
         return row_count, error_list
 
-    def get_data_types_from_object(self, object : DatabaseObject, column_intersections: list) -> dict:
-        """ returns datatypes for all intersection columns in a database object 
+    def get_data_types_from_object(self, object: DatabaseObject, column_intersections: list) -> dict:
+        """returns datatypes for all intersection columns in a database object
 
         Args:
             object (DatabaseObject): table or view
@@ -264,20 +270,22 @@ class SnowflakeService(object):
             self._connect_to_snowflake()
 
         column_intersections = str(column_intersections)[1:-1]
-        if column_intersections == '':
+        if column_intersections == "":
             column_intersections = "''"
 
-        query_get_data_types_from_object=f"SELECT COLUMN_NAME , DATA_TYPE   \
+        query_get_data_types_from_object = f"SELECT COLUMN_NAME , DATA_TYPE   \
                                             FROM {object.database.upper()}.INFORMATION_SCHEMA.COLUMNS  \
                                             WHERE TABLE_NAME='{object.name.upper()}'  \
                                             AND TABLE_SCHEMA = '{object.schema.upper()}'  \
                                             AND COLUMN_NAME IN ({column_intersections})  \
                                             ;"
 
-        dict_colummns_datatype=self.execute_queries(query_get_data_types_from_object)
+        dict_colummns_datatype = self.execute_queries(query_get_data_types_from_object)
         return dict_colummns_datatype
 
-    def get_count_distincts_from_object(self, object: DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns: list=[]) -> dict:
+    def get_count_distincts_from_object(
+        self, object: DatabaseObject, column_intersections: list, where_clause: str = "", exclude_columns: list = []
+    ) -> dict:
         """get distinct count for every column in a database object that is in column intersections list
 
         Args:
@@ -294,27 +302,26 @@ class SnowflakeService(object):
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        unions=""
+        unions = ""
 
         for column in column_intersections:
             if column not in exclude_columns:
-                unions +=f" UNION SELECT '{column}' AS COLUMN_NAME, COUNT(DISTINCT {column}) AS COUNT_DISTINCT FROM {object.database}.{object.schema}.{object.name} {where_clause}"
+                unions += f" UNION SELECT '{column}' AS COLUMN_NAME, COUNT(DISTINCT {column}) AS COUNT_DISTINCT FROM {object.database}.{object.schema}.{object.name} {where_clause}"
 
-        query_get_count_distincts_from_object=f"{unions[6:]} ORDER BY COUNT_DISTINCT;"
+        query_get_count_distincts_from_object = f"{unions[6:]} ORDER BY COUNT_DISTINCT;"
         error_list = []
         try:
-            dict_count_distincts=self.execute_queries(query_get_count_distincts_from_object)
-        
+            dict_count_distincts = self.execute_queries(query_get_count_distincts_from_object)
+
         except Exception as err:
-            #raise err
-            dict_count_distincts = [{'COUNT_DISTINCT': 0}]
-            error_list.append(["ERROR", str(err).split('|||')[0], str(err).split('|||')[1]])
+            # raise err
+            dict_count_distincts = [{"COUNT_DISTINCT": 0}]
+            error_list.append(["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]])
 
-        
         return dict_count_distincts, error_list
 
-    def get_table_size(self, object : DatabaseObject) -> int:
-        """ returns size of given object 
+    def get_table_size(self, object: DatabaseObject) -> int:
+        """returns size of given object
 
         Args:
             object (DatabaseObject): table or view
@@ -332,8 +339,15 @@ class SnowflakeService(object):
 
         return size
 
-    def create_checksums(self, object : DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns:list=[], numeric_scale: int = None) -> List[Dict]:
-        """ creates checksums for given object in compliance with given conditions
+    def create_checksums(
+        self,
+        object: DatabaseObject,
+        column_intersections: list,
+        where_clause: str = "",
+        exclude_columns: list = [],
+        numeric_scale: int = None,
+    ) -> list[dict]:
+        """creates checksums for given object in compliance with given conditions
 
         Args:
             object (DatabaseObject): table or view
@@ -362,66 +376,67 @@ class SnowflakeService(object):
             count_nulls += f", SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END) AS COUNTNULLS_{column}"
 
             if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
-
                 if numeric_scale:
-                    aggregates += f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS sum_{column}"
+                    aggregates += (
+                        f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS sum_{column}"
+                    )
                 else:
                     aggregates += f", CAST(SUM({column}) AS DECIMAL(38)) AS sum_{column}"
 
             elif (
-                column_datatype.lower()  in self.snowflake_datatype_mapping["string"] 
-                or column_datatype.lower()  in self.snowflake_datatype_mapping["date_and_time"]
+                column_datatype.lower() in self.snowflake_datatype_mapping["string"]
+                or column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]
             ):
-
                 aggregates += f", COUNT(DISTINCT LOWER({column})) AS countdistinct_{column}"
 
             elif column_datatype.lower() in self.snowflake_datatype_mapping["binary"]:
-
                 aggregates += f", COUNT(DISTINCT LOWER(TRY_TO_NUMBER({column}::VARCHAR))) AS countdistinct_{column}"
 
             elif column_datatype.lower() in self.snowflake_datatype_mapping["boolean"]:
-
                 aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = false) :: VARCHAR AS aggregateboolean_{column}"
 
-            #else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
+            # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
-        query_checksums = f"SELECT {aggregates[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+        query_checksums = (
+            f"SELECT {aggregates[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+        )
 
-        query_countnulls = f"SELECT {count_nulls[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+        query_countnulls = (
+            f"SELECT {count_nulls[1:]} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
+        )
 
         error_list = []
-        test_list=[]
-        aggregation_results={}
+        test_list = []
+        aggregation_results = {}
 
         try:
-            checksums_results = self.execute_queries([query_checksums,query_countnulls])
+            checksums_results = self.execute_queries([query_checksums, query_countnulls])
 
-            aggregation_results=checksums_results[0][0]
+            aggregation_results = checksums_results[0][0]
 
-            countnulls_results=checksums_results[1][0]
+            countnulls_results = checksums_results[1][0]
 
-            for i in range(0,len(aggregation_results)):
-                
+            for i in range(0, len(aggregation_results)):
                 if list(aggregation_results.values())[i] is None:
                     agg_result = 0
                 else:
                     agg_result = list(aggregation_results.values())[i]
-                
+
                 if list(countnulls_results.values())[i] is None:
                     cnt_result = 0
                 else:
                     cnt_result = list(countnulls_results.values())[i]
 
-
-                test_list.append([[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i],agg_result,cnt_result])
+                test_list.append(
+                    [[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i], agg_result, cnt_result]
+                )
 
         except Exception as err:
-            error_list.append(["ERROR", str(err).split('|||')[0], str(err).split('|||')[1]])
+            error_list.append(["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]])
 
+        checksums = dict(zip([item.split("_", 1)[1] for item in aggregation_results.keys()], test_list))
+        checksums["TESTATM_ERRORS"] = error_list
 
-        checksums = dict(zip([item.split("_", 1)[1] for item in aggregation_results.keys()] , test_list))
-        checksums['TESTATM_ERRORS'] = error_list
-        
         return checksums
 
     def create_pandas_df_from_group_by(
@@ -434,8 +449,8 @@ class SnowflakeService(object):
         only_numeric: bool,
         where_clause: str,
         exclude_columns: list,
-        numeric_scale: int = None
-    ) -> List[Dict]:
+        numeric_scale: int = None,
+    ) -> list[dict]:
         """execution of multiple aggregations at once
 
         Args:
@@ -450,16 +465,24 @@ class SnowflakeService(object):
             numeric_scale (int, optional): number of decimal places for aggregations. Defaults to None.
 
         Returns:
-            List[Dict]: list of pandas dataframes with results from aggregations, used sql queries 
+            List[Dict]: list of pandas dataframes with results from aggregations, used sql queries
         """
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
         if group_by_aggregation_columns == ["all"]:
-            aggregation_columns= [f"{column.upper()}" for column in column_intersections if (column not in group_by_columns and column not in exclude_columns)]
+            aggregation_columns = [
+                f"{column.upper()}"
+                for column in column_intersections
+                if (column not in group_by_columns and column not in exclude_columns)
+            ]
         else:
-            aggregation_columns= [f"{column.upper()}" for column in column_intersections if (column in group_by_aggregation_columns and column not in exclude_columns)]
+            aggregation_columns = [
+                f"{column.upper()}"
+                for column in column_intersections
+                if (column in group_by_aggregation_columns and column not in exclude_columns)
+            ]
 
         group_by_query_columns_string = " "
         grouping_columns_final = []
@@ -473,16 +496,15 @@ class SnowflakeService(object):
 
             group_by_query_columns_string = group_by_query_columns_string[:-1]
 
-            dict_colummns_datatype=self.get_data_types_from_object(object, aggregation_columns)
+            dict_colummns_datatype = self.get_data_types_from_object(object, aggregation_columns)
 
             aggregates = ""
             aggregates_min = ""
 
             for column in aggregation_columns:
+                column_datatype = next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
 
-                column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
-
-                if column_datatype.lower() in  self.snowflake_datatype_mapping["numeric"]:
+                if column_datatype.lower() in self.snowflake_datatype_mapping["numeric"]:
                     if numeric_scale:
                         aggregates_min += f", CAST(ROUND(MIN({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MIN_{column}, CAST(ROUND(max({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MAX_{column}"
                         aggregates += f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS SUM_{column}"
@@ -490,19 +512,19 @@ class SnowflakeService(object):
                         aggregates_min += f", MIN({column}) AS MIN_{column}, MAX({column}) AS MAX_{column}"
                         aggregates += f", SUM({column}) AS SUM_{column}"
 
-                elif not only_numeric and (column_datatype.lower() in  self.snowflake_datatype_mapping["string"] or column_datatype.lower() in  self.snowflake_datatype_mapping["date_and_time"]):
-
+                elif not only_numeric and (
+                    column_datatype.lower() in self.snowflake_datatype_mapping["string"]
+                    or column_datatype.lower() in self.snowflake_datatype_mapping["date_and_time"]
+                ):
                     aggregates += f", COUNT(DISTINCT LOWER({column})) AS COUNTDISTINCT_{column}"
 
-                elif not only_numeric and column_datatype.lower() in  self.snowflake_datatype_mapping["binary"]:
-
+                elif not only_numeric and column_datatype.lower() in self.snowflake_datatype_mapping["binary"]:
                     aggregates += f", COUNT(DISTINCT LOWER(TRY_TO_NUMBER({column}::VARCHAR))) AS COUNTDISTINCT_{column}"
 
-                elif not only_numeric and column_datatype.lower() in  self.snowflake_datatype_mapping["boolean"]:
-
+                elif not only_numeric and column_datatype.lower() in self.snowflake_datatype_mapping["boolean"]:
                     aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column} = false) :: VARCHAR AS AGGREGATEBOOLEAN_{column}"
-                
-                #else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
+
+                # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
             # CASE 1: min_max
             if group_by_aggregation_type == "only_min_max":
@@ -515,35 +537,44 @@ class SnowflakeService(object):
             # CASE 3: sum, count_distinct, aggregate_boolean, min_max
             elif group_by_aggregation_type == "various_and_min_max":
                 group_by_query_aggregation_string = f"{aggregates_min[1:]}{aggregates}"
-        
+
             query_group_by_aggregation = f"SELECT {group_by_query_columns_string}, COUNT(*) AS COUNT_OF_GROUP_BY_VALUE, {group_by_query_aggregation_string} FROM {object.database}.{object.schema}.{object.name} {where_clause} GROUP BY {group_by_query_columns_string} ORDER BY {group_by_query_columns_string};"
 
-            group_by_aggregation_pdf = self.execute_queries(query_group_by_aggregation,True)
+            group_by_aggregation_pdf = self.execute_queries(query_group_by_aggregation, True)
         except Exception as err:
             group_by_aggregation_pdf = pd.DataFrame()
             group_by_aggregation_pdf["TESTATM_ERROR"] = [1]
             if not grouping_columns_final:
                 error_dict = {
                     "QUERY": "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table",
-                    "ERROR":  "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table"
+                    "ERROR": "NO Group-BY Columns found in the Columns Intersection. Please check if the configurated Group-By Columns exist in the Table",
                 }
                 group_by_query_aggregation_string = ""
-            elif '|||' in str(err):
-                error_dict = {
-                    "QUERY": str(err).split('|||')[0],
-                    "ERROR":  str(err).split('|||')[1]
-                }
+            elif "|||" in str(err):
+                error_dict = {"QUERY": str(err).split("|||")[0], "ERROR": str(err).split("|||")[1]}
             else:
                 error_dict = {
                     "QUERY": "NO Query generated. Please check if the configurated Grouping Columns exist in the Table",
-                    "ERROR":  str(err)
+                    "ERROR": str(err),
                 }
                 group_by_query_aggregation_string = ""
 
-        return group_by_aggregation_pdf, group_by_query_aggregation_string, group_by_query_columns_string, grouping_columns_final, error_dict
+        return (
+            group_by_aggregation_pdf,
+            group_by_query_aggregation_string,
+            group_by_query_columns_string,
+            grouping_columns_final,
+            error_dict,
+        )
 
-    def create_pandas_df(self, object : DatabaseObject, intersection_columns_trgt_src: list, where_clause:str="", exclude_columns:list=[]) -> pd.DataFrame:
-        """ creates pandas dataframes with all data from given object in given columns
+    def create_pandas_df(
+        self,
+        object: DatabaseObject,
+        intersection_columns_trgt_src: list,
+        where_clause: str = "",
+        exclude_columns: list = [],
+    ) -> pd.DataFrame:
+        """creates pandas dataframes with all data from given object in given columns
 
         Args:
             object (DatabaseObject): table or view
@@ -556,16 +587,26 @@ class SnowflakeService(object):
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
-        intersection_columns_trgt_src_ = ', '.join(list(set(intersection_columns_trgt_src) - set(exclude_columns)))
+        intersection_columns_trgt_src_ = ", ".join(list(set(intersection_columns_trgt_src) - set(exclude_columns)))
 
         df_query = f"SELECT {intersection_columns_trgt_src_} FROM {object.database}.{object.schema}.{object.name} {where_clause};"
-            
-        src_pdf = self.execute_queries(df_query,True)
+
+        src_pdf = self.execute_queries(df_query, True)
 
         return src_pdf
 
-    def create_pandas_df_from_sample(self, object: DatabaseObject, column_intersections: list, key_columns: list, where_clause:str="", exclude_columns:list=[], key_filters: dict={}, dedicated_columns: list=[], sample_count :int=10, numeric_scale: int = None) -> List[Dict]:
- 
+    def create_pandas_df_from_sample(
+        self,
+        object: DatabaseObject,
+        column_intersections: list,
+        key_columns: list,
+        where_clause: str = "",
+        exclude_columns: list = [],
+        key_filters: dict = {},
+        dedicated_columns: list = [],
+        sample_count: int = 10,
+        numeric_scale: int = None,
+    ) -> list[dict]:
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
 
@@ -579,34 +620,37 @@ class SnowflakeService(object):
         dedicated_intersection.sort()
 
         if not where_clause:
-            where_clause= 'WHERE 1=1 '
+            where_clause = "WHERE 1=1 "
 
         if dedicated_intersection != []:
             is_dedicated = True
 
-            dict_colummns_datatype=self.get_data_types_from_object(object, dedicated_intersection)
+            dict_colummns_datatype = self.get_data_types_from_object(object, dedicated_intersection)
 
         else:
             is_dedicated = False
 
-            dict_colummns_datatype=self.get_data_types_from_object(object, column_intersections)
+            dict_colummns_datatype = self.get_data_types_from_object(object, column_intersections)
 
-       
         if key_intersection != [] and is_dedicated:
             keys = str(key_intersection)[1:-1].replace("'", "")
-            column_clause, numeric_columns, used_columns = self._get_column_clause(dedicated_intersection, dict_colummns_datatype, numeric_scale, key_columns)
+            column_clause, numeric_columns, used_columns = self._get_column_clause(
+                dedicated_intersection, dict_colummns_datatype, numeric_scale, key_columns
+            )
             if (key_filters != {}) & (filter_intersection != []):
                 values = list(key_filters.values())
                 if values[0] != []:
-                    in_clause = self._get_in_clause(key_filters, numeric_columns, numeric_scale) 
+                    in_clause = self._get_in_clause(key_filters, numeric_columns, numeric_scale)
                 else:
                     in_clause = ""
             else:
-                in_clause = "" 
+                in_clause = ""
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause}{in_clause} ORDER BY {keys};"
         elif key_intersection != [] and not is_dedicated:
             keys = str(key_intersection)[1:-1].replace("'", "")
-            column_clause, numeric_columns, used_columns = self._get_column_clause(column_intersections, dict_colummns_datatype, numeric_scale, key_columns)
+            column_clause, numeric_columns, used_columns = self._get_column_clause(
+                column_intersections, dict_colummns_datatype, numeric_scale, key_columns
+            )
             if (key_filters != {}) & (filter_intersection != []):
                 values = list(key_filters.values())
                 if values[0] != []:
@@ -617,9 +661,11 @@ class SnowflakeService(object):
                 in_clause = ""
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause}{in_clause} ORDER BY {keys};"
         else:
-            column_intersections = list(set(column_intersections)  - set(exclude_columns))
+            column_intersections = list(set(column_intersections) - set(exclude_columns))
             column_intersections.sort()
-            column_clause, numeric_columns, used_columns = self._get_column_clause(column_intersections, dict_colummns_datatype, numeric_scale, key_columns)
+            column_clause, numeric_columns, used_columns = self._get_column_clause(
+                column_intersections, dict_colummns_datatype, numeric_scale, key_columns
+            )
             sample_query = f"SELECT {column_clause} FROM {object.database}.{object.schema}.{object.name} SAMPLE ({sample_count} ROWS) {where_clause};"
 
         error_dict = {}
@@ -635,26 +681,21 @@ class SnowflakeService(object):
         except Exception as err:
             sample_pdf = pd.DataFrame()
             sample_pdf["TESTATM_ERROR"] = [1]
-            if '|||' in str(err):
-                error_dict = {
-                    "QUERY": str(err).split('|||')[0],
-                    "ERROR":  str(err).split('|||')[1]
-                }
+            if "|||" in str(err):
+                error_dict = {"QUERY": str(err).split("|||")[0], "ERROR": str(err).split("|||")[1]}
             else:
-                error_dict = {
-                    "QUERY": 'No SQL Error',
-                    "ERROR":  str(err)
-                }
+                error_dict = {"QUERY": "No SQL Error", "ERROR": str(err)}
 
         return_list = []
         return_list.append(sample_pdf)
         return_list.append(error_dict)
 
+        return return_list, key_dict, used_columns, sample_query
 
-        return return_list , key_dict, used_columns, sample_query
-
-    def execute_queries(self, query: Union[str, List[str]],return_as_pdf:bool=False, return_query_ids:bool=False)  -> Union[List[Dict], List[List[Dict]]]:
-        """ actual execution of defined queries
+    def execute_queries(
+        self, query: str | list[str], return_as_pdf: bool = False, return_query_ids: bool = False
+    ) -> list[dict] | list[list[dict]]:
+        """actual execution of defined queries
 
         Args:
             query (Union[str, List[str]]): queries to be executed
@@ -670,23 +711,23 @@ class SnowflakeService(object):
 
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-        
+
         if query:
-            query_list: List[str] = query if isinstance(query, list) else [query]
+            query_list: list[str] = query if isinstance(query, list) else [query]
         else:
-            logger.error('Query defined as null - please check input for execute_queries function.')
+            logger.error("Query defined as null - please check input for execute_queries function.")
 
-        cursor = self.snowflake_connection.cursor(snowflake.connector.DictCursor)
+        cursor = self.snowflake_connection.get_connection_object().cursor(snowflake.connector.DictCursor)
 
         results = []
-        query_ids=[]
+        query_ids = []
 
         for single_query in query_list:
-            try:            
+            try:
                 query_result = cursor.execute(single_query).fetchall()
                 if return_as_pdf:
                     query_result = pd.DataFrame(query_result)
-                
+
                 results.append(query_result)
                 query_ids.append(cursor.sfqid)
 
@@ -699,7 +740,7 @@ class SnowflakeService(object):
         else:
             return results[0] if not isinstance(query, list) else results
 
-    def execute_statement(self, statement: Union[str, List[str]]) -> None:
+    def execute_statement(self, statement: str | list[str]) -> None:
         """
             Executes simple statement against snowflake
             Schema and Database settings must be set beforehand
@@ -708,23 +749,18 @@ class SnowflakeService(object):
         """
         if self.snowflake_connection is None:
             self._connect_to_snowflake()
-            
-        statement_list: List[str] = (
-            statement if isinstance(statement, list) else [statement]
-        )
+
+        statement_list: list[str] = statement if isinstance(statement, list) else [statement]
 
         try:
             for single_statement in statement_list:
-                stripped_statement = (
-                    single_statement.strip()
-                )  
-                _ = self.snowflake_connection.execute_string(stripped_statement)
+                stripped_statement = single_statement.strip()
+                _ = self.snowflake_connection.get_connection_object().execute_string(stripped_statement)
 
         except Exception as err:
             raise Exception(self._get_error_message(err, single_statement)) from err
-        
+
     def upload_to_stage(self, stage_name: str, folder_path: str, file_name: str, is_temporary: bool):
-        
         file_path = PurePath(folder_path).joinpath(PurePath(file_name))
 
         if is_temporary:
@@ -734,48 +770,70 @@ class SnowflakeService(object):
 
         put_query = rf"PUT 'file://{file_path}' @{stage_name};"
 
-        put_query = put_query.replace("\\","\\\\")
+        put_query = put_query.replace("\\", "\\\\")
 
         self.execute_statement(create_query)
 
         self.execute_statement(put_query)
 
-    def insert_json_results(self, run_guid: str, pipeline_name: str, pipeline_id: str, start_time_utc: str, result_table: str, stage_name: str ) -> None: 
+    def insert_json_results(
+        self,
+        run_guid: str,
+        pipeline_name: str,
+        pipeline_id: str,
+        start_time_utc: str,
+        result_table: str,
+        stage_name: str,
+    ) -> None:
         """
-            copy into - result table for json results
+        copy into - result table for json results
         """
-        result_database = result_table.split('.')[0]
-        meta_data_schema = result_table.split('.')[1]
+        result_database = result_table.split(".")[0]
+        meta_data_schema = result_table.split(".")[1]
 
         statement = f"COPY INTO {result_table} (RUN_GUID, PIPELINE_NAME, PIPELINE_ID, START_TIME_UTC, RESULT, CREATION_TIME_UTC) FROM (SELECT '{run_guid}', '{pipeline_name}', '{pipeline_id}', '{start_time_utc}', $1, SYSDATE() from @{stage_name} (file_format => {result_database}.{meta_data_schema}.ff_json ));"
 
         self.execute_statement(statement)
 
-    def insert_json_results_live(self, run_guid: str, pipeline_name: str, pipeline_id: str,  result_table: str, stage_name: str , source_system: str, target_system:str, database:str, schema:str, object:str) -> None: 
+    def insert_json_results_live(
+        self,
+        run_guid: str,
+        pipeline_name: str,
+        pipeline_id: str,
+        result_table: str,
+        stage_name: str,
+        source_system: str,
+        target_system: str,
+        database: str,
+        schema: str,
+        object: str,
+    ) -> None:
         """
-            copy into - result table for json results live
+        copy into - result table for json results live
         """
-        result_database = result_table.split('.')[0]
-        meta_data_schema = result_table.split('.')[1]
+        result_database = result_table.split(".")[0]
+        meta_data_schema = result_table.split(".")[1]
 
         statement = f"COPY INTO {result_table} (RUN_GUID, PIPELINE_NAME, PIPELINE_ID, SOURCE_SYSTEM, TARGET_SYSTEM, DATABASE_NAME, SCHEMA_NAME, OBJECT_NAME  ,RESULT, CREATION_TS) FROM (SELECT '{run_guid}', '{pipeline_name}', '{pipeline_id}', '{source_system}', '{target_system}', '{database}', '{schema}', '{object}', $1, SYSDATE() from @{stage_name} (file_format => {result_database}.{meta_data_schema}.ff_json ));"
 
         self.execute_statement(statement)
 
-    def insert_highlevel_results(self, results: dict, run_guid:str, pipeline_name: str, pipeline_id: str, result_table_highlevel: str) -> None: 
+    def insert_highlevel_results(
+        self, results: dict, run_guid: str, pipeline_name: str, pipeline_id: str, result_table_highlevel: str
+    ) -> None:
         """
-            insert into - highlevel results per "pipeline run" / "ics data validation execution"
+        insert into - highlevel results per "pipeline run" / "ics data validation execution"
         """
-        TESTSET_ = ', '.join(results['TESTSET'])
+        TESTSET_ = ", ".join(results["TESTSET"])
 
-        OBJECTS_TO_COMPARE_SRC_ = ', '.join(results['OBJECTS_TO_COMPARE_SRC'])
+        OBJECTS_TO_COMPARE_SRC_ = ", ".join(results["OBJECTS_TO_COMPARE_SRC"])
 
-        OBJECTS_TO_COMPARE_TRGT_ = ', '.join(results['OBJECTS_TO_COMPARE_TRGT'])
+        OBJECTS_TO_COMPARE_TRGT_ = ", ".join(results["OBJECTS_TO_COMPARE_TRGT"])
 
-        SRC_MINUS_TRGT_ = ', '.join(results['SRC_MINUS_TRGT'])
+        SRC_MINUS_TRGT_ = ", ".join(results["SRC_MINUS_TRGT"])
+
+        TRGT_MINUS_SRC_ = ", ".join(results["TRGT_MINUS_SRC"])
 
-        TRGT_MINUS_SRC_ = ', '.join(results['TRGT_MINUS_SRC'])
-        
         insert_statement = f"INSERT INTO {result_table_highlevel} ( \
                                                                         RUN_GUID, \
                                                                         PIPELINE_NAME,  \
@@ -819,13 +877,13 @@ class SnowflakeService(object):
                                                                         '{results['NUMBER_OF_OBJECTS_TO_COMPARE']}',  \
                                                                         '{SRC_MINUS_TRGT_}',  \
                                                                         '{TRGT_MINUS_SRC_}', \
-                                                                        SYSDATE())"  
-                                                                        
+                                                                        SYSDATE())"
+
         self.execute_statement(insert_statement)
 
-    def insert_objectlevel_results(self, result_table: dict, result_table_objectlevel: str, run_guid:str) -> None:
+    def insert_objectlevel_results(self, result_table: dict, result_table_objectlevel: str, run_guid: str) -> None:
         """
-            insert into - detailed results per object
+        insert into - detailed results per object
         """
         insert_statement = f"INSERT INTO {result_table_objectlevel} ( \
                                                                         RUN_GUID, \
@@ -954,15 +1012,14 @@ class SnowflakeService(object):
                                             FROM {result_table} RESULTS \
                                             CROSS JOIN LATERAL FLATTEN(INPUT => RESULT:OBJECTS) F1\
                                             WHERE RUN_GUID = '{run_guid}'\
-                                ;" 
+                                ;"
 
         self.execute_statement(insert_statement)
 
-
-    def insert_columnlevel_results(self, result_table: str, result_table_columnlevel: str, run_guid:str) -> None: 
+    def insert_columnlevel_results(self, result_table: str, result_table_columnlevel: str, run_guid: str) -> None:
+        """
+        insert into - detailed results per column
         """
-            insert into - detailed results per column
-        """   
         insert_statement = f"INSERT INTO {result_table_columnlevel} ( \
                                                                 RUN_GUID,\
                                                                 PIPELINE_ID,\
@@ -1039,5 +1096,5 @@ class SnowflakeService(object):
                                                             CROSS JOIN LATERAL FLATTEN(INPUT => RESULT:OBJECTS) F1\
                                                             CROSS JOIN LATERAL FLATTEN(INPUT => F1.VALUE:COLUMNS) F2\
                                                             WHERE RUN_GUID = '{run_guid}';"
-        
-        self.execute_statement(insert_statement)
+
+        self.execute_statement(insert_statement)

icsDataValidation-1.0.361.dist-info/METADATA

@@ -0,0 +1,21 @@
+Metadata-Version: 2.2
+Name: icsDataValidation
+Version: 1.0.361
+Summary: Add your description here
+Home-page: https://initions.com/
+Author: initions
+Author-email: ICSMC_EXT_PYPIORG@accenture.com
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: azure-storage-blob==12.13.1
+Requires-Dist: boto3==1.26.154
+Requires-Dist: cloe-util-snowflake-connector==1.0.5
+Requires-Dist: databricks-sdk==0.29.0
+Requires-Dist: databricks-sql-connector==3.0.1
+Requires-Dist: numpy==1.26.3
+Requires-Dist: oracledb==2.5.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: pyexasol==0.24.0
+Requires-Dist: pyodbc
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: teradatasql==17.20.0.10

{icsDataValidation-1.0.360.dist-info → icsDataValidation-1.0.361.dist-info}/RECORD

@@ -1,5 +1,3 @@
-examples/ics_data_validation.py,sha256=vyBAnU8yQGKGH33ZxrvaZpY-kt1iQ3h53kzkKG0Y7gI,139
-examples/manual_execution_params.template.py,sha256=g3LAah1zEXJtozAZFpkxCm-JCWXSQY3R2SG-8YcPV9c,2038
 icsDataValidation/configuration.py,sha256=HOFjmC8_e2nvoItndMtJQQA1MR5aCgZGeF1AwY_FvjE,477
 icsDataValidation/main.py,sha256=nmbFM8Epf4-Nhd9ArH31wT7Yx0MSjIHxX93zPke1ArA,12498
 icsDataValidation/connection_setups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -7,7 +5,7 @@ icsDataValidation/connection_setups/azure_connection_setup.py,sha256=gvTyctG63ol
 icsDataValidation/connection_setups/databricks_connection_setup.py,sha256=dNEBum-8R-TUW2SCEk3CaNtCr_gLFvn456KBlENpgJU,1220
 icsDataValidation/connection_setups/exasol_connection_setup.py,sha256=RfCUsL6G-NaOW-qNK-3SfHcljbRaKD6fDIHXkNQhClk,590
 icsDataValidation/connection_setups/oracle_connection_setup.py,sha256=D-4ucC1ChE4HYm93ECIEg_yBOrn1NkknxFBgFRGFmWs,978
-icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=JDTdIM0bQ0_IV0HnCct90RC6Mq4fS1sfh9IJc-YYlMo,804
+icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=IgEhni4Q0oYGh2QzptpyfEUvUt3cVO28jNSGg11cxyI,1778
 icsDataValidation/connection_setups/teradata_connection_setup.py,sha256=fIpuxz-FTqFK2vSMSuokqU9sdJkaJ4UP5piY_zIbj5k,624
 icsDataValidation/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/core/database_objects.py,sha256=2oaDaVQajSYI_HJjJy1pmc6FsoK_wMfwgu6ZgEcFvow,523
@@ -28,7 +26,7 @@ icsDataValidation/services/database_services/databricks_hive_metastore_service.p
 icsDataValidation/services/database_services/databricks_unity_catalog_service.py,sha256=INA8rd3KW_jAplNagGa9tEON3dyOufcIAPOOdmc0Mrc,70259
 icsDataValidation/services/database_services/exasol_service.py,sha256=7LYnRScO3DxBmuSN0HmTgsFc2el-Ii3A9jgGsXSJVU8,11074
 icsDataValidation/services/database_services/oracle_service.py,sha256=60unwWlHm520ioFmz0y2K8ApwZrruf9iB0ojjQx0IWc,31523
-icsDataValidation/services/database_services/snowflake_service.py,sha256=UWmjQZN4oX2ctH6uhE2oklXGHo66SK5UnQbFYFhzDuc,60630
+icsDataValidation/services/database_services/snowflake_service.py,sha256=EYOZjkjeh0CMGApef-LWoXP4JeJzhAG_qUCqpwOQ9ek,61021
 icsDataValidation/services/database_services/teradata_service.py,sha256=Rf0xzcZGEbooq3r2Rfe2fCahTm2Xw4uznQa8vyWoyqM,40169
 icsDataValidation/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/utils/file_util.py,sha256=ZTMB1sTnIIdffg9tEJRCFQQ5SG8Fksc5ie1PM4gHXG4,3432
@@ -36,7 +34,7 @@ icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI
 icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
 icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
 icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
-icsDataValidation-1.0.360.dist-info/METADATA,sha256=FwmapeuYYE6bQD6M1d5RvtyTA-d82Cg-bfkTmuyHqwo,720
-icsDataValidation-1.0.360.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-icsDataValidation-1.0.360.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
-icsDataValidation-1.0.360.dist-info/RECORD,,
+icsDataValidation-1.0.361.dist-info/METADATA,sha256=m3_7gnPsag7iS3Kg02_JjMV7azhFZqR4H6nUK70dlnw,24605
+icsDataValidation-1.0.361.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+icsDataValidation-1.0.361.dist-info/top_level.txt,sha256=BqWUGJb4J7ZybpDMeuGHxEHGHwXXJEIURd9pBybHzTM,18
+icsDataValidation-1.0.361.dist-info/RECORD,,

{icsDataValidation-1.0.360.dist-info → icsDataValidation-1.0.361.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

icsDataValidation-1.0.361.dist-info/top_level.txt

@@ -0,0 +1 @@
+icsDataValidation

examples/ics_data_validation.py

@@ -1,7 +0,0 @@
-
-from  manual_execution_params import manual_execution_params
-from icsDataValidation import main
-
-manual_execution_params()
-
-main.execute()

examples/manual_execution_params.template.py

@@ -1,44 +0,0 @@
-#########################################################################################
-#########################################################################################
-import os
-
-def manual_execution_params():
-
-    # Manual execution: File location of the icsDataValidation configuration
-    os.environ["CONFIG_FOLDER_NAME"]                    = 'examples/'
-    os.environ["CONFIGURATION_FILE_NAME"]               = 'ics_data_validation_config.json'
-    os.environ["MIGRATION_CONFIGURATION_FILE_NAME"]     = 'migration_config.json'
-
-    # Manual execution: File path of the locally stored secrets
-    # Syntax: <parameter_name>="<value>" per row
-    os.environ["ENV_FILEPATH"]  = ''
-
-    # Manual execution: Testset settings
-    os.environ["DATABASE_NAME"] = '' #
-    os.environ["SCHEMA_NAME"] = '' #
-
-    os.environ["TESTSET_FILE_NAMES"] = ''  # for no testset define as ''
-
-    os.environ["OBJECT_TYPE_RESTRICTION"] = '' #'include_all', 'include_only_tables', 'include_only_views'
-
-    # Manual execution: Result settings
-    os.environ["UPLOAD_RESULT_TO_BLOB"] = '' #boolean: True or False
-    os.environ["UPLOAD_RESULT_TO_BUCKET"] = '' #boolean: True or False
-    os.environ["UPLOAD_RESULT_TO_RESULT_DATABASE"] = ''#boolean: True or False
-
-    # Manual execution: Pandas Dataframe Comparison restrictions -> -1 for no pandas-df comparison at all
-    os.environ["MAX_OBJECT_SIZE"] = str(-1) #-1
-    os.environ["MAX_ROW_NUMBER"]  = str(-1) #-1
-
-    # Manual execution: Parallelization of comparison settings
-    os.environ["MAX_NUMBER_OF_THREADS"]  = str(1) #1
-
-    # Manual execution: Group-By-Aggregation settings
-    os.environ["EXECUTE_GROUP_BY_COMPARISON"] = '' #boolean: True or False
-    os.environ["USE_GROUP_BY_COLUMNS"] = '' #boolean: True or False
-    os.environ["MIN_GROUP_BY_COUNT_DISTINCT"] = str(2) #2
-    os.environ["MAX_GROUP_BY_COUNT_DISTINCT"] = str(5) #5
-    os.environ["MAX_GROUP_BY_SIZE"] = str(100000000) #100000000
-
-    # Manual execution: Precision settings
-    os.environ["NUMERIC_SCALE"] = str(2)

icsDataValidation-1.0.360.dist-info/METADATA

@@ -1,20 +0,0 @@
-Metadata-Version: 2.1
-Name: icsDataValidation
-Version: 1.0.360
-Summary: ics data validation
-Home-page: https://initions.com/
-Author: initions
-Author-email: ICSMC_EXT_PYPIORG@accenture.com
-License: MIT
-Requires-Dist: snowflake-connector-python[pandas] (==3.7.1)
-Requires-Dist: python-dotenv (==0.20.0)
-Requires-Dist: pyodbc
-Requires-Dist: pyexasol (==0.24.0)
-Requires-Dist: pandas (==2.2.2)
-Requires-Dist: azure-storage-blob (==12.13.1)
-Requires-Dist: teradatasql (==17.20.0.10)
-Requires-Dist: boto3 (==1.26.154)
-Requires-Dist: oracledb (==2.5.0)
-Requires-Dist: databricks-sql-connector (==3.0.1)
-Requires-Dist: databricks-sdk (==0.29.0)
-Requires-Dist: numpy (==1.26.3)

icsDataValidation-1.0.360.dist-info/top_level.txt

@@ -1,4 +0,0 @@
-azure-pipelines
-examples
-icsDataValidation
-resources