icsDataValidation 1.0.232__py3-none-any.whl

examples/ics_data_validation.py

@@ -0,0 +1,7 @@
+
+from  manual_execution_params import manual_execution_params
+from icsDataValidation import main
+
+manual_execution_params()
+
+main.execute()

examples/manual_execution_params.template.py

@@ -0,0 +1,44 @@
+#########################################################################################
+#########################################################################################
+import os
+
+def manual_execution_params():
+
+    # Manual execution: File location of the icsDataValidation configuration
+    os.environ["CONFIG_FOLDER_NAME"]                    = 'examples/'
+    os.environ["CONFIGURATION_FILE_NAME"]               = 'ics_data_validation_config.json'
+    os.environ["MIGRATION_CONFIGURATION_FILE_NAME"]     = 'migration_config.json'
+
+    # Manual execution: File path of the locally stored secrets
+    # Syntax: <parameter_name>="<value>" per row
+    os.environ["ENV_FILEPATH"]  = ''
+
+    # Manual execution: Testset settings
+    os.environ["DATABASE_NAME"] = '' #
+    os.environ["SCHEMA_NAME"] = '' #
+
+    os.environ["TESTSET_FILE_NAMES"] = ''  # for no testset define as ''
+
+    os.environ["OBJECT_TYPE_RESTRICTION"] = '' #'include_all', 'include_only_tables', 'include_only_views'
+
+    # Manual execution: Result settings
+    os.environ["UPLOAD_RESULT_TO_BLOB"] = '' #boolean: True or False
+    os.environ["UPLOAD_RESULT_TO_BUCKET"] = '' #boolean: True or False
+    os.environ["UPLOAD_RESULT_TO_RESULT_DATABASE"] = ''#boolean: True or False
+
+    # Manual execution: Pandas Dataframe Comparison restrictions -> -1 for no pandas-df comparison at all
+    os.environ["MAX_OBJECT_SIZE"] = str(-1) #-1
+    os.environ["MAX_ROW_NUMBER"]  = str(-1) #-1
+
+    # Manual execution: Parallelization of comparison settings
+    os.environ["MAX_NUMBER_OF_THREADS"]  = str(1) #1
+
+    # Manual execution: Group-By-Aggregation settings
+    os.environ["EXECUTE_GROUP_BY_COMPARISON"] = '' #boolean: True or False
+    os.environ["USE_GROUP_BY_COLUMNS"] = '' #boolean: True or False
+    os.environ["MIN_GROUP_BY_COUNT_DISTINCT"] = str(2) #2
+    os.environ["MAX_GROUP_BY_COUNT_DISTINCT"] = str(5) #5
+    os.environ["MAX_GROUP_BY_SIZE"] = str(100000000) #100000000
+
+    # Manual execution: Precision settings
+    os.environ["NUMERIC_SCALE"] = str(2)

icsDataValidation/configuration.py

@@ -0,0 +1,19 @@
+import os
+from typing import Dict, List, Union
+
+import utils.file_util as file_util
+
+class icsDataValidationConfig(object):
+    """
+    Holds icsDataValidation config.
+    """
+
+    def __init__(self):
+        """ """
+        self.module_root_folder = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "..")
+        )
+
+        self.config_path = os.environ.get("ICSDATAVALIDATION_CONFIG_PATH")
+
+        self.config_dict = file_util.load_json(self.config_path)

icsDataValidation/connection_setups/azure_connection_setup.py

@@ -0,0 +1,19 @@
+import os
+
+from dotenv import load_dotenv
+from pathlib import Path
+
+#########################################################################################
+#########################################################################################
+
+def load_azure_credentials(system_configs:dict,system_selection:str)->dict:
+
+    azure_params = {
+        "Server"    : system_configs[system_selection]["SERVER"],
+        "Database"  : system_configs[system_selection]["DATABASE"],
+        "User"      : system_configs[system_selection]["USER"],
+        "Password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"]),
+        "Driver"    : system_configs[system_selection]["DRIVER"],
+    }
+
+    return azure_params

icsDataValidation/connection_setups/databricks_connection_setup.py

@@ -0,0 +1,28 @@
+#########################################################################################
+#########################################################################################
+
+from databricks.sdk.core import Config, oauth_service_principal
+import os
+
+def load_databricks_credentials(database_configs: dict, system_selection: str) -> dict:
+
+    if "DBX_ACCESS_TOKEN_NAME" in database_configs[system_selection]:
+        # personal access token authentication
+        access_token = os.getenv(database_configs[system_selection]["DBX_ACCESS_TOKEN_NAME"])
+    else:
+        # OAuth machine-to-machine (M2M) authentication
+        ad_config = {
+            "tenant_id": database_configs[system_selection]["TENANT_ID"],
+            "client_id": database_configs[system_selection]["CLIENT_ID"],
+            "client_secret": os.getenv(database_configs[system_selection]["CLIENT_SECRET"]),
+        }
+
+        access_token=oauth_service_principal(Config(ad_config))
+
+    databricks_params = {
+        "server_hostname": database_configs[system_selection]["SERVER_HOSTNAME"],
+        "http_path": database_configs[system_selection]["CLUSTER_HTTP_PATH"],
+        "access_token": access_token
+    }
+
+    return databricks_params

icsDataValidation/connection_setups/exasol_connection_setup.py

@@ -0,0 +1,17 @@
+import os
+
+from dotenv import load_dotenv
+from pathlib import Path
+
+#########################################################################################
+#########################################################################################
+
+def load_exasol_credentials(system_configs:dict,system_selection:str)->dict:
+
+    exasol_params = {
+        "dsn"       : system_configs[system_selection]["DSN"],
+        "user"      : system_configs[system_selection]["USER"],
+        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"])
+    }
+
+    return exasol_params

icsDataValidation/connection_setups/oracle_connection_setup.py

@@ -0,0 +1,26 @@
+import os
+import oracledb
+
+from dotenv import load_dotenv
+from pathlib import Path
+
+#########################################################################################
+#########################################################################################
+
+def load_oracle_credentials(system_configs:dict,system_selection:str)->dict:
+
+    oracle_params = {
+        "user"   : system_configs[system_selection]["USERNAME"],
+        "dsn"      : system_configs[system_selection]["DSN"],
+        "port" : system_configs[system_selection]["PORT"],
+        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"])
+    }
+
+    if "SERVICE_NAME" in system_configs[system_selection]:
+        oracle_params["service_name"] = system_configs[system_selection]["SERVICE_NAME"]
+
+    if "MODE" in system_configs[system_selection]:
+        mode = system_configs[system_selection]["MODE"]
+        oracle_params["mode"] = getattr(oracledb, mode)
+
+    return oracle_params

icsDataValidation/connection_setups/snowflake_connection_setup.py

@@ -0,0 +1,20 @@
+import os
+
+from dotenv import load_dotenv
+from pathlib import Path
+
+#########################################################################################
+#########################################################################################
+
+def load_snowflake_credentials(system_configs:dict,system_selection:str)->dict:
+
+    snowflake_params = {
+        "account"   : system_configs[system_selection]["ACCOUNT"],
+        "user"      : system_configs[system_selection]["USER"],
+        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"]),
+        "warehouse" : system_configs[system_selection]["WAREHOUSE"],
+        "role"      : system_configs[system_selection]["ROLE"],
+        "database"  : system_configs[system_selection]["DATABASE"]
+    }
+
+    return snowflake_params

icsDataValidation/connection_setups/teradata_connection_setup.py

@@ -0,0 +1,18 @@
+import os
+
+from dotenv import load_dotenv
+from pathlib import Path
+
+#########################################################################################
+#########################################################################################
+
+def load_teradata_credentials(system_configs:dict,system_selection:str)->dict:
+
+    teradata_params = {
+        "host"   : system_configs[system_selection]["HOST"],
+        "user"      : system_configs[system_selection]["USER"],
+        "password"  : os.getenv(system_configs[system_selection]["PASSWORD_NAME"]),
+        "dbs_port"  : '1025'
+    }
+
+    return teradata_params

icsDataValidation/core/database_objects.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+class DatabaseObjectType(Enum):
+    """
+    Enum for valid database object types
+    """
+
+    TABLE = "table"
+    VIEW = "view"
+
+class DatabaseObject():
+    def __init__(self, object_identifier: str, object_type: DatabaseObjectType):
+        self.identifier = object_identifier
+        self.database   = object_identifier.split(".",2)[0]
+        self.schema     = object_identifier.split(".",2)[1]
+        self.name       = object_identifier.split(".",2)[2]
+        self.type       = object_type
+    

icsDataValidation/core/object_comparison.py

@@ -0,0 +1,218 @@
+import logging
+import time
+
+from typing import Union, List, Dict 
+from threading import current_thread
+from pathlib import PurePath
+
+from icsDataValidation.services.system_service import SystemService
+from icsDataValidation.services.comparison_service import ComparisonService
+from icsDataValidation.services.result_service import prepare_column_level_result, prepare_object_level_result, prepare_object_level_live_result
+from icsDataValidation.utils.sql_util import parse_filter
+from icsDataValidation.utils.logger_util import configure_dev_ops_logger
+from icsDataValidation.utils.file_util import write_json_to_file
+from icsDataValidation.core.database_objects import DatabaseObject
+from icsDataValidation.input_parameters.testing_tool_params import TestingToolParams
+
+#########################################################################################
+# Configure Dev Ops Logger
+
+logger = logging.getLogger('Object_Comparison')
+logger.setLevel(logging.INFO)
+configure_dev_ops_logger(logger)
+
+#########################################################################################
+#########################################################################################
+
+def get_additional_configuration(src_object: DatabaseObject, testing_tool_params: TestingToolParams) -> Union[str, List[str]]:
+    """
+    Get additional configuration from the migration_config.json. Retrieve e.g. the filter and excluded columns.
+    """
+    global_filter = ""
+    exclude_columns = []
+    if "ADDITIONAL_CONFIGURATION" in testing_tool_params.migration_config.keys():
+        additional_configuration = testing_tool_params.migration_config["ADDITIONAL_CONFIGURATION"]
+        if f"{src_object.database}.{src_object.schema}.{src_object.name}" in additional_configuration.keys():
+            if "FILTER" in additional_configuration[f"{src_object.database}.{src_object.schema}.{src_object.name}"]:
+                global_filter = parse_filter(additional_configuration[f"{src_object.database}.{src_object.schema}.{src_object.name}"]["FILTER"])
+                logger.info(f"FILTER: {global_filter} ")
+            if "EXCLUDE_COLUMNS" in additional_configuration[f"{src_object.database}.{src_object.schema}.{src_object.name}"]:
+                exclude_columns = additional_configuration[f"{src_object.database}.{src_object.schema}.{src_object.name}"]["EXCLUDE_COLUMNS"]
+                exclude_columns = [excluded_column.upper() for excluded_column in exclude_columns]
+                logger.info(f"EXCLUDE_COLUMNS: {exclude_columns} ")
+    
+    return global_filter, exclude_columns
+
+def compare_objects(testing_tool_params: TestingToolParams, objects_to_compare: List[Dict]) -> List[Dict]:
+
+    source_system=SystemService(testing_tool_params.source_system_selection,testing_tool_params.systems)
+    target_system=SystemService(testing_tool_params.target_system_selection,testing_tool_params.systems)
+    result_system=SystemService(testing_tool_params.result_system_selection,testing_tool_params.systems)
+
+    testing_tool_params.connection_params_src=source_system.get_connection_params()
+    testing_tool_params.connection_params_trgt=target_system.get_connection_params()
+    testing_tool_params.connection_params_result = result_system.get_connection_params()
+
+    database_service_src=source_system.initialize_database_service(testing_tool_params.connection_params_src)
+    database_service_trgt=target_system.initialize_database_service(testing_tool_params.connection_params_trgt)
+    database_service_result=result_system.initialize_database_service(testing_tool_params.connection_params_result)
+
+    with database_service_src as db_service_src, database_service_trgt as db_service_trgt, database_service_result as db_service_result:
+
+        object_level_comparison_results=[]
+
+        for n_object, object in enumerate(objects_to_compare):
+            start_time_object_comparison_ = time.time()
+            comp_id = n_object+1
+            #####################################################################
+            # initialize comparison service
+
+            src_object=DatabaseObject(object["src_object_identifier"],object["src_object_type"])
+            trgt_object=DatabaseObject(object["trgt_object_identifier"],object["trgt_object_type"])
+
+            logger.info(f"++++++++++++++++ [{comp_id}] START Comparison of {src_object.database}.{src_object.schema}.{src_object.name} vs. {trgt_object.database}.{trgt_object.schema}.{trgt_object.name}")
+
+            global_filter, exclude_columns = get_additional_configuration(src_object, testing_tool_params)
+
+            comparison_service=ComparisonService(src_object, trgt_object, db_service_src, db_service_trgt, global_filter, exclude_columns, comp_id)
+        
+            #####################################################################
+            # execute comparison
+
+            comparison_service.row_count_comparison()
+            comparison_service.column_names_comparison()
+            comparison_service.aggregation_comparison()
+            if testing_tool_params.execute_group_by_comparison:
+                comparison_service.group_by_comparison()
+            comparison_service.sample_comparison()
+            comparison_service.pandas_dataframe_comparison()
+
+            #####################################################################
+            # TODO as function - check if the object was changed during comparison 
+
+            ### structure of output needs to be adjusted to enable comparison using > in the if statements
+            ### florian said the feature is not too important for now, so it's being skipped for now
+
+            # comparison_service.result_params.last_altered_src = db_service_src.get_last_altered_timestamp_from_object(src_object)
+            # last_altered_trgt = db_service_trgt.get_last_altered_timestamp_from_object(trgt_object)
+
+            # if comparison_service.result_params.last_altered_src>start_time_utc:
+            #     comparison_service.result_params.not_altered_during_comparison_src = False   
+
+            # if last_altered_trgt>start_time_utc:
+            #     comparison_service.result_params.not_altered_during_comparison_trgt = False
+
+            #####################################################################
+            # prepare column level results
+
+            comparison_service.result_params.all_count_nulls_equal = True
+            column_level_comparison_results = []
+
+            for column in comparison_service.result_params.all_columns_trgt_src:
+
+                column_level_comparison_result=prepare_column_level_result(column, exclude_columns, comparison_service.result_params)
+
+                if column_level_comparison_result["COUNT_NULLS_EQUAL"] is False:
+                    comparison_service.result_params.all_count_nulls_equal = False
+                
+                column_level_comparison_results.append(column_level_comparison_result)
+
+            #####################################################################
+            # prepare object level result
+
+            object_level_comparison_result = prepare_object_level_result(
+                src_object, 
+                trgt_object, 
+                global_filter, 
+                exclude_columns, 
+                comparison_service.result_params, 
+                column_level_comparison_results
+            )
+
+            object_level_comparison_results.append(object_level_comparison_result)
+
+            #####################################################################
+            # prepare and upload live result of the current object
+
+            live_object_level_comparison_result = prepare_object_level_live_result(
+                    object_level_comparison_result, 
+                    testing_tool_params, 
+                )
+
+            # TODO write as function
+            if testing_tool_params.upload_result_to_result_database and not (testing_tool_params.upload_result_to_result_database =='upload_result_to_result_database env variable not found' or testing_tool_params.upload_result_to_result_database =='False'):
+                
+                stage_name = f'{testing_tool_params.result_database_name}.{testing_tool_params.stage_schema}."STG_LIVE_{src_object.schema}_{src_object.name}_{testing_tool_params.run_guid}"'
+                
+                result_file_name = f"Comparison_Result_Live_{src_object.schema}_{src_object.name}.json"
+
+                result_file_path = testing_tool_params.live_result_folder_path.joinpath(PurePath(result_file_name))
+
+                write_json_to_file(live_object_level_comparison_result, result_file_path)
+
+                # TODO handle result systems other than Snowflake
+                if testing_tool_params.systems[testing_tool_params.result_system_selection]["DATABASE_TYPE"] == 'snowflake':
+
+                    db_service_result.upload_to_stage(stage_name, testing_tool_params.live_result_folder_path, result_file_name, False)
+
+                    db_service_result.insert_json_results_live(testing_tool_params.run_guid, testing_tool_params.pipeline_name, testing_tool_params.pipeline_id, testing_tool_params.result_live_table, stage_name, testing_tool_params.source_system_selection, testing_tool_params.target_system_selection, testing_tool_params.database_name, src_object.schema, src_object.name)
+
+            end_time_object_comparison_ = time.time()
+            #####################################################################
+            # object level result log
+
+            # TODO write as function
+            logger.info('****************************************************')
+            logger.info(f"++++++++++++++++ [{comp_id}] Comparison Result: {comp_id} of {len(objects_to_compare)} ++++++++++++++++")
+            logger.info(f"[{comp_id}] Source object => {object['src_object_identifier']}")
+            logger.info(f"[{comp_id}] Target object => {object['trgt_object_identifier']}")
+            logger.info(f"[{comp_id}] --- Comparison Time: {round(end_time_object_comparison_ - start_time_object_comparison_, 2)} s")
+            if  comparison_service.result_params.row_counts_equal:
+                logger.info(f"[{comp_id}] --- Row counts --------> EQUAL")
+            else:
+                logger.info(f"[{comp_id}] --- Row counts --------> NOT equal")
+                logger.info(f"[{comp_id}] --- Source row count: {comparison_service.result_params.src_row_count}. Target row count: {comparison_service.result_params.trgt_row_count}")
+            
+            if len(comparison_service.result_params.src_columns_upper) != len(set(comparison_service.result_params.src_columns_upper)):
+                logger.info(f"[{comp_id}] --- Duplicates in the source column names -> The source system seems to be case sensitive.")
+
+            if len(comparison_service.result_params.trgt_columns_upper) != len(set(comparison_service.result_params.trgt_columns_upper)):
+                logger.info(f"[{comp_id}] --- Duplicates in the target column names -> The target system seems to be case sensitive.")
+
+            if comparison_service.result_params.columns_equal:
+                logger.info(f"[{comp_id}] --- Column names ------> EQUAL")
+            else:
+                logger.info(f"[{comp_id}] --- Column names ------> NOT equal")
+                logger.info(f"[{comp_id}] ------ src_minus_trgt {comparison_service.result_params.src_columns_minus_trgt_columns} and trgt_minus_src {comparison_service.result_params.trgt_columns_minus_src_columns}")
+
+            if comparison_service.result_params.aggregations_equal:
+                logger.info(f"[{comp_id}] --- Aggregations ------> EQUAL")
+            else:
+                logger.info(f"[{comp_id}] --- Aggregations ------> NOT equal")
+
+            if not comparison_service.result_params.object_group_by_columns:
+                logger.info(f"[{comp_id}] --- Group-By ----------> NOT compared")
+            elif comparison_service.result_params.group_by_equal:
+                logger.info(f"[{comp_id}] --- Group-By ----------> EQUAL")
+            else:
+                logger.info(f"[{comp_id}] --- Group-By ----------> NOT equal")
+
+            if not comparison_service.result_params.samples_compared:
+                logger.info(f"[{comp_id}] --- Samples -----------> NOT compared")
+            elif comparison_service.result_params.samples_equal:
+                logger.info(f"[{comp_id}] --- Samples -----------> EQUAL")
+            else:
+                logger.info(f"[{comp_id}] --- Samples -----------> NOT equal")
+
+            if not comparison_service.result_params.pandas_df_compared:
+                logger.info(f"[{comp_id}] --- Pandas Dataframes -> NOT compared")
+                logger.info(f"[{comp_id}] ------ src_tbl_size: {comparison_service.result_params.src_tbl_size} trgt_tbl_size:{comparison_service.result_params.trgt_tbl_size} max_object_size {testing_tool_params.max_object_size}")
+                logger.info(f"[{comp_id}] ------ src_row_count: {comparison_service.result_params.src_row_count} trgt_row_count:{comparison_service.result_params.trgt_row_count} max_row_number {testing_tool_params.max_row_number}")
+            elif comparison_service.result_params.pandas_df_is_equal:
+                logger.info(f"[{comp_id}] --- Pandas Dataframes -> EQUAL")
+            
+            else:
+                logger.info(f"[{comp_id}] --- Pandas Dataframes -> NOT equal")
+            logger.info('****************************************************')
+
+    return object_level_comparison_results

icsDataValidation/input_parameters/testing_tool_params.py

@@ -0,0 +1,81 @@
+#########################################################################################
+#########################################################################################
+
+import os
+import uuid
+
+from dataclasses import dataclass
+from dotenv import load_dotenv
+from pathlib import Path, PurePath
+
+
+#########################################################################################
+#########################################################################################
+
+@dataclass
+class TestingToolParams:
+
+    pipeline_id: str                            = os.environ.get('BUILD_BUILDNUMBER')
+    pipeline_name: str                          = os.environ.get('BUILD_DEFINITIONNAME','build_definitionname env variable not found')
+
+    #########################################################################################
+    
+    # manual execution load input parameters
+    if pipeline_id is None:
+        from examples.manual_execution_params import manual_execution_params
+
+        pipeline_name = 'manual'
+
+        manual_execution_params()
+
+        env_filepath: str = os.environ.get('ENV_FILEPATH', 'env_filepath env variable not found')
+
+        # load in env variables from local file (e.g. passwords or azure blob storage connection string )
+        _ = load_dotenv(dotenv_path=(PurePath(Path.home()).joinpath(PurePath(env_filepath))), override=True)
+
+    #########################################################################################
+    config_folder_name:str                      = os.environ.get('CONFIG_FOLDER_NAME', 'CONFIG FOLDER NAME env variable not found')
+    configuration_file_name:str                 = os.environ.get('CONFIGURATION_FILE_NAME', 'DATABASE CONFIGURATION FILE NAME env variable not found')
+    migration_configuration_file_name: str      = os.environ.get('MIGRATION_CONFIGURATION_FILE_NAME', 'MIGRATION CONFIGURATION FILE NAME env variable not found')
+
+    database_name: str                          = None if os.environ.get('DATABASE_NAME','Database name env variable not found') == 'null' else os.environ.get('DATABASE_NAME','Database name env variable not found')
+    schema_name: str                            = None if os.environ.get('SCHEMA_NAME','Target schema name env variable not found') == 'null' else os.environ.get('SCHEMA_NAME','Target schema name env variable not found')
+    testset_file_names: str                     = os.environ.get('TESTSET_FILE_NAMES','testset_file_names env variable not found')
+    object_type_restriction: str                = os.environ.get('OBJECT_TYPE_RESTRICTION','object_type_restriction env variable not found')
+    azure_devops_pipeline: bool                 = True if os.environ.get('AZURE_DEVOPS_PIPELINE','azure_devops_pipeline env variable not found') == 'True' else False
+    gitlab_pipeline: bool                       = True if os.environ.get('GITLAB_PIPELINE','gitlab_pipeline env variable not found') == 'True' else False
+    upload_result_to_blob: bool                 = True if os.environ.get('UPLOAD_RESULT_TO_BLOB','upload_result_to_blob env variable not found') == 'True' else False
+    upload_result_to_bucket: bool               = True if os.environ.get('UPLOAD_RESULT_TO_BUCKET','upload_result_to_bucket env variable not found') == 'True' else False
+    upload_result_to_result_database: bool      = True if os.environ.get('UPLOAD_RESULT_TO_RESULT_DATABASE','upload_result_to_result_database env variable not found') == 'True' else False
+    max_object_size: int                        = int(os.environ.get('MAX_OBJECT_SIZE','max_object_size env variable not found'))
+    max_row_number: int                         = int(os.environ.get('MAX_ROW_NUMBER','max_row_number env variable not found'))
+    max_number_of_threads: int                  = int(os.environ.get('MAX_NUMBER_OF_THREADS','max_number_of_threads env variable not found'))
+    execute_group_by_comparison: bool           = True if os.environ.get('EXECUTE_GROUP_BY_COMPARISON','execute_group_by_comparison env variable not found') == 'True' else False
+    use_group_by_columns: bool                  = True if os.environ.get('USE_GROUP_BY_COLUMNS','use_group_by_columns env variable not found') == 'True' else False
+    min_group_by_count_distinct: int            = int(os.environ.get('MIN_GROUP_BY_COUNT_DISTINCT','min_group_by_count_distinct env variable not found'))
+    max_group_by_count_distinct: int            = int(os.environ.get('MAX_GROUP_BY_COUNT_DISTINCT','max_group_by_count_distinct env variable not found'))
+    max_group_by_size: int                      = int(os.environ.get('MAX_GROUP_BY_SIZE','max_group_by_size env variable not found'))
+    numeric_scale: int                          = int(os.environ.get('NUMERIC_SCALE','numeric_scale env variable not found'))
+    branch_name: str                            = os.environ.get('BRANCH_NAME', 'branch_name env variable not found')
+    source_branch:str                           = os.environ.get('BUILD_SOURCEBRANCH', 'build_sourcebranch env variable not found')  
+    azure_storage_connection_string: str        = os.environ.get('AZURE_STORAGE_CONNECTION_STRING','azure_storage_connection_string env variable not found')
+    aws_bucket_access_key: str                  = os.environ.get('AWS_BUCKET_ACCESS_KEY', 'aws_bucket_access_key env variable not found')
+    aws_bucket_secret_key: str                  = os.environ.get('AWS_BUCKET_SECRET_KEY', 'aws_bucket_secret_key env variable not found')
+    run_guid: str                               = str(uuid.uuid4())
+    testatm_access_token: str                   = os.environ.get('TESTATM_ACCESS_TOKEN', 'testatm_access_token env variable not found')
+    gitlab_ci_server_host: str                  = os.environ.get('GITLAB_CI_SERVER_HOST', 'gitlab_ci_server_host env variable not found')
+    gitlab_ci_project_path: str                 = os.environ.get('GITLAB_CI_PROJECT_PATH', 'gitlab_ci_project_path env variable not found')
+    
+
+    #########################################################################################
+
+    if max_object_size != 'max_object_size env variable not found':
+        max_object_size=int(max_object_size)
+    if max_row_number != 'max_row_number env variable not found':
+        max_row_number=int(max_row_number)
+    if max_number_of_threads != 'max_number_of_threads env variable not found':
+        max_number_of_threads=int(max_number_of_threads)
+    if min_group_by_count_distinct != 'min_group_by_count_distinct env variable not found':
+        min_group_by_count_distinct=int(min_group_by_count_distinct)
+    if max_group_by_count_distinct != 'max_group_by_count_distinct env variable not found':
+        max_group_by_count_distinct=int(max_group_by_count_distinct)