icsDataValidation 1.0.358__py3-none-any.whl

icsDataValidation/services/system_service.py

@@ -0,0 +1,61 @@
+from icsDataValidation.connection_setups.snowflake_connection_setup import load_snowflake_credentials
+from icsDataValidation.connection_setups.exasol_connection_setup import load_exasol_credentials
+from icsDataValidation.connection_setups.azure_connection_setup import load_azure_credentials
+from icsDataValidation.connection_setups.teradata_connection_setup import load_teradata_credentials
+from icsDataValidation.connection_setups.oracle_connection_setup import load_oracle_credentials
+from icsDataValidation.connection_setups.databricks_connection_setup import load_databricks_credentials
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+from icsDataValidation.services.database_services.teradata_service import TeradataService
+from icsDataValidation.services.database_services.exasol_service import ExasolService
+from icsDataValidation.services.database_services.azure_service import AzureService
+from icsDataValidation.services.database_services.oracle_service import OracleService
+from icsDataValidation.services.database_services.databricks_hive_metastore_service import DatabricksHiveMetastoreService
+from icsDataValidation.services.database_services.databricks_unity_catalog_service import DatabricksUnityCatalogService
+
+#########################################################################################
+#########################################################################################
+
+class SystemService:
+    """
+    Class to initialize database services dependent on the system selection.
+    """
+
+    def __init__(self, system_selection: str, database_config: dict):
+        self.database_config = database_config
+        self.system_selection = system_selection
+        self.system_type = database_config[system_selection]["DATABASE_TYPE"].upper()
+
+    def get_connection_params(self):
+        """
+        Get the connection parameters dependent on the system type.
+        """
+        credentials_function_mapping = {
+            "SNOWFLAKE": load_snowflake_credentials,
+            "EXASOL": load_exasol_credentials,
+            "AZURE": load_azure_credentials,
+            "TERADATA": load_teradata_credentials,
+            "ORACLE": load_oracle_credentials,
+            "DATABRICKS_HIVE_METASTORE": load_databricks_credentials,
+            "DATABRICKS_UNITY_CATALOG": load_databricks_credentials,
+        }
+
+        connection_params = credentials_function_mapping[self.system_type](
+            self.database_config, self.system_selection
+        )
+        return connection_params
+
+    def initialize_database_service(self, connection_params: dict):
+        """
+        Initialize the database service dependent on the system type.
+        """
+        database_service_mapping = {
+            "SNOWFLAKE": SnowflakeService,
+            "EXASOL": ExasolService,
+            "AZURE": AzureService,
+            "TERADATA": TeradataService,
+            "ORACLE": OracleService,
+            "DATABRICKS_HIVE_METASTORE": DatabricksHiveMetastoreService,
+            "DATABRICKS_UNITY_CATALOG": DatabricksUnityCatalogService,
+        }
+        database_service = database_service_mapping[self.system_type](connection_params)
+        return database_service

icsDataValidation/services/testset_service.py

@@ -0,0 +1,257 @@
+import logging
+
+from icsDataValidation.utils.logger_util import configure_dev_ops_logger
+
+#########################################################################################
+# Configure Dev Ops Logger
+
+logger = logging.getLogger('TestsetService')
+logger.setLevel(logging.INFO)
+configure_dev_ops_logger(logger)
+
+#########################################################################################
+#########################################################################################
+
+class TestsetService:
+    """
+    Class to prepare the set of objects for the comparison.
+    Maps schemas and objects between source and target.
+    Handles blacklists and whitelists.
+    """
+
+    def __init__(self, testset_mapping:dict, testset_blacklist: dict, testset_whitelist: dict=None):
+        self.testset_mapping = testset_mapping
+        self.testset_blacklist = testset_blacklist
+        self.testset_whitelist = testset_whitelist
+
+    def handle_database_mapping(self, source_database_name: str = None) -> str:
+        """
+        Map the source and the target database. 
+        Note: Case-Insensitive and returns upper-case target database name.
+        """
+        target_database_name=source_database_name.upper()
+
+        if self.testset_mapping and "DATABASE_MAPPING" in self.testset_mapping:
+            for database_mapping in self.testset_mapping["DATABASE_MAPPING"]:
+                if source_database_name.upper() == database_mapping["src_database_name"].upper():
+                    target_database_name = database_mapping["trgt_database_name"].upper()
+
+        return target_database_name
+
+    def handle_schema_mapping(self, source_schema_name: str = None, source_database_name: str = None) -> str:
+        """
+        Map the source and the target schema. 
+        Note: Case-Insensitive and returns upper-case target schema name.
+        """
+        target_schema_name=source_schema_name.upper()
+        found_schema_mapping = False
+
+        if self.testset_mapping and "SCHEMA_MAPPING" in self.testset_mapping:
+            for schema_mapping in self.testset_mapping["SCHEMA_MAPPING"]:
+                
+                if f"{source_database_name.upper()}.{source_schema_name.upper()}" == schema_mapping["src_schema_identifier"].upper():
+                    target_schema_name = schema_mapping["trgt_schema_name"].upper()
+                    found_schema_mapping = True
+
+        return target_schema_name, found_schema_mapping
+
+    def handle_schema_replace_mapping(self, source_schema_name: str = None) -> str:
+        """
+        Map the source and the target schema by replacing a subset of the target schema string.
+        Note: Case-Insensitive and returns upper-case target schema name.
+        """
+
+        if self.testset_mapping and "SCHEMA_REPLACE_MAPPING" in self.testset_mapping:
+            replace_mapping = self.testset_mapping["SCHEMA_REPLACE_MAPPING"]
+            for replace_object in replace_mapping:
+                target_schema_name = source_schema_name.upper().replace(
+                    replace_object["src_replace_value"].upper(),
+                    replace_object["trgt_replace_value"].upper(),
+                )
+        else:
+            target_schema_name=source_schema_name.upper()
+
+        return target_schema_name
+    
+    def handle_blacklist(self, database_objects: dict, src_trgt: str)-> dict:
+        """
+        Handle the blacklist from the migration_config to restrict database objects.
+        Use src_trgt="SRC" for source and src_trgt="TRGT" for target.
+        """
+        blacklist_objects=[object_blacklisted.upper() for object_blacklisted in self.testset_blacklist[f"BLACKLIST_OBJECTS_{src_trgt}"]]
+        blacklist_schemas=[schema_blacklisted.upper() for schema_blacklisted in self.testset_blacklist[f"BLACKLIST_SCHEMAS_{src_trgt}"]]
+        blacklist_databases=[database_blacklisted.upper() for database_blacklisted in self.testset_blacklist[f"BLACKLIST_DATABASES_{src_trgt}"]]
+
+        database_objects_=database_objects.copy()
+
+        for db_object in database_objects_:
+            database_name = db_object["object_identifier"].split(".",1)[0]
+            schema_identifier = ".".join(db_object["object_identifier"].split(".",2)[:2])
+            if database_name in blacklist_databases:
+                database_objects.remove(db_object)
+            elif schema_identifier in blacklist_schemas:
+                database_objects.remove(db_object)
+            elif db_object["object_identifier"] in blacklist_objects:
+                database_objects.remove(db_object)
+
+        return database_objects
+    
+    def handle_whitelist(self, database_objects: dict, src_trgt: str)-> dict:
+        """
+        Handle the whitelist which is defined as a testset to restrict database objects.
+        Use src_trgt="SRC"  for source and src_trgt="TRGT" for target.
+        """
+        whitelist_objects=[object_whitelisted.upper() for object_whitelisted in self.testset_whitelist[f"WHITELIST_OBJECTS_{src_trgt}"]]
+        whitelist_schemas=[schema_whitelisted.upper() for schema_whitelisted in self.testset_whitelist[f"WHITELIST_SCHEMAS_{src_trgt}"]]
+        whitelist_databases=[database_whitelisted.upper() for database_whitelisted in self.testset_whitelist[f"WHITELIST_DATABASES_{src_trgt}"]]
+
+        database_objects_=database_objects.copy()
+
+        for db_object in database_objects_:
+            database_name = db_object["object_identifier"].split(".",1)[0]
+            schema_identifier = ".".join(db_object["object_identifier"].split(".",2)[:2])
+            if not db_object["object_identifier"].upper() in whitelist_objects and schema_identifier.upper() not in whitelist_schemas and database_name.upper() not in whitelist_databases:
+                database_objects.remove(db_object)
+
+        return database_objects
+    
+    def map_objects(self, database_objects_src: list, database_objects_trgt: list):
+        """
+        Maps objects between source and target by using the mapping defined in the migration_config.json.
+        Handles object "1:1"-mapping  and object "replace"-mapping.
+        Returns remaining_mapping_objects which differ between source and target and can not be mapped.
+        Returns a flag all_objects_matching which indicates if there exist remaining_mapping_objects.
+        """
+        intersection_objects_mapped_trgt_src = []
+        remaining_mapping_objects = []
+        src_objects_minus_trgt_objects = [object for object in database_objects_src if object not in database_objects_trgt]
+        trgt_objects_minus_src_objects = [object for object in database_objects_trgt if object not in database_objects_src]
+
+
+        trgt_objects_minus_src_table_identifiers = [object["object_identifier"] for object in database_objects_trgt if object not in database_objects_src and object["object_type"] == 'table']
+        trgt_objects_minus_src_view_identifiers = [object["object_identifier"] for object in database_objects_trgt if object not in database_objects_src and object["object_type"] == 'view']
+        
+
+        if database_objects_src != database_objects_trgt and self.testset_mapping:
+ 
+            src_objects_minus_trgt_objects_ = src_objects_minus_trgt_objects.copy()
+
+            trgt_objects_minus_src_object_identifiers=[object["object_identifier"] for object in trgt_objects_minus_src_objects]
+
+            for n_db_object, db_object in enumerate(src_objects_minus_trgt_objects_):
+                logger.info(f"Object {n_db_object+1} of {len(src_objects_minus_trgt_objects_)}: {db_object}")
+                continue_flag = True
+
+                #########################################################################################
+                # Object-Mapping
+                for mapping in self.testset_mapping["OBJECT_MAPPING"]:
+                
+                    if (
+                        db_object["object_identifier"] == mapping["src_object_identifier"].upper() 
+                        and db_object["object_type"] == mapping["src_object_type"] 
+                        and mapping['trgt_object_identifier'].upper() in trgt_objects_minus_src_object_identifiers
+                    ):
+                        logger.info(f" -> mapping object found: {mapping}")
+                        intersection_objects_mapped_trgt_src.append({"src_object_identifier": db_object["object_identifier"],"src_object_type": db_object["object_type"], "trgt_object_identifier": mapping["trgt_object_identifier"],"trgt_object_type": mapping["trgt_object_type"]})
+                        src_objects_minus_trgt_objects.remove(db_object)
+
+                        for trgt_object in trgt_objects_minus_src_objects:
+                            if trgt_object["object_identifier"] == mapping["trgt_object_identifier"].upper():
+                                trgt_objects_minus_src_objects.remove(trgt_object)
+                        logger.info(" -> added by 1:1 mapping")
+                        
+                        # set continue_flag to false because this object has been covered by the mapping 
+                        continue_flag = False
+                        break
+
+                ##########################################################################################
+                # Database-Mapping, and Schema-Mapping
+
+                if continue_flag == True:
+                    
+                    src_database_name = db_object["object_identifier"].split(".",1)[0]
+                    src_schema_name = db_object["object_identifier"].split(".",2)[1]
+                    src_object_name = db_object["object_identifier"].split(".",2)[2]
+
+                    trgt_database_name=self.handle_database_mapping(src_database_name)
+                    trgt_schema_name, _ =self.handle_schema_mapping(src_schema_name,src_database_name)
+
+                    trgt_object_identifier=f"{trgt_database_name}.{trgt_schema_name}.{src_object_name}".upper()
+
+                    if (db_object["object_type"] == 'table' and trgt_object_identifier in trgt_objects_minus_src_table_identifiers) or (db_object["object_type"] == 'view' and trgt_object_identifier in trgt_objects_minus_src_view_identifiers):
+                        intersection_objects_mapped_trgt_src.append({"src_object_identifier": db_object["object_identifier"],"src_object_type": db_object["object_type"], "trgt_object_identifier": trgt_object_identifier,"trgt_object_type": db_object["object_type"]})
+                        src_objects_minus_trgt_objects.remove(db_object)
+                        
+                        for trgt_object in trgt_objects_minus_src_objects:
+                            if trgt_object["object_identifier"] == trgt_object_identifier:
+                                trgt_objects_minus_src_objects.remove(trgt_object)
+
+                        logger.info(" -> added by database/schema-mapping")
+
+                        # set continue_flag to false because this object has been covered by the replacements 
+                        continue_flag = False
+
+                ##########################################################################################
+                # Replace-Mapping
+
+                if continue_flag == True:
+                    
+
+                        src_database_name = db_object["object_identifier"].split(".",1)[0]
+                        src_schema_name = db_object["object_identifier"].split(".",2)[1]
+                        src_object_name = db_object["object_identifier"].split(".",2)[2]
+
+                        #TODO rework!!!!
+
+                        ## replace the values from the migration_config.json to create a potential_match which can be looked for the trgt_objects_minus_src_objects list
+                        #potential_match = db_object["object_identifier"].upper().replace(f'{substitute["src_replace_value"].upper()}',f'{substitute["trgt_replace_value"].upper()}')
+#
+                        ## the potential_match is contained within the trgt_objects_minus_src_objects list
+                        #if potential_match in trgt_objects_minus_src_object_identifiers:
+                        #    logger.info(f" -> replace mapping found: {substitute}")
+                        #    intersection_objects_mapped_trgt_src.append({"src_object_identifier": db_object["object_identifier"],"src_object_type": db_object["object_type"], "trgt_object_identifier": potential_match,"trgt_object_type": db_object["object_type"]})
+                        #    src_objects_minus_trgt_objects.remove(db_object)
+#
+                        #    for trgt_object in trgt_objects_minus_src_objects:
+                        #        if trgt_object["object_identifier"] == potential_match:
+                        #            trgt_objects_minus_src_objects.remove(trgt_object)
+                        #    logger.info(" -> added by replace mapping")
+#
+                        #    # set continue_flag to false because this object has been covered by the replacements 
+                        #    continue_flag = False
+                        #    break
+
+                #####################################################################
+                # Remaining objects
+                if continue_flag == True:
+                    remaining_mapping_objects.append({"src_object_identifier": db_object["object_identifier"],"trgt_object_identifier": '',"src_object_type": db_object["object_type"],"trgt_object_type": ''})
+                    logger.info(" -> no mapping found -> added to remaining_mapping_objects")
+
+        object_identifiers_src_minus_trgt= [object["object_identifier"] for object in src_objects_minus_trgt_objects]
+        object_identifiers_trgt_minus_src= [object["object_identifier"] for object in trgt_objects_minus_src_objects]
+
+        if src_objects_minus_trgt_objects:
+            logger.warning('There are database objects in the source db that are not in the target db and for which no mapping exists:')
+            logger.warning(f"{object_identifiers_src_minus_trgt}")
+        if trgt_objects_minus_src_objects:
+            logger.warning('There are database objects in the target db that are not in the source db and for which no mapping exists:')
+            logger.warning(f"{object_identifiers_trgt_minus_src}")
+
+        if not (src_objects_minus_trgt_objects and trgt_objects_minus_src_objects):
+            all_objects_matching=True
+        else:
+            all_objects_matching=False
+
+        return intersection_objects_mapped_trgt_src, object_identifiers_src_minus_trgt, object_identifiers_trgt_minus_src, remaining_mapping_objects, all_objects_matching
+        
+    @staticmethod
+    def get_intersection_objects_trgt_src(database_objects_src: list, database_objects_trgt: list, intersection_objects_mapped_trgt_src:list):
+        """
+        Get intersection of all database objects from source db and target db - including mapped objects.
+        """
+
+        intersection_objects_trgt_src_without_mapping =[{"src_object_identifier": object["object_identifier"],"src_object_type": object["object_type"],"trgt_object_identifier": object["object_identifier"],"trgt_object_type": object["object_type"]} for object in database_objects_src if object in database_objects_trgt]
+
+        intersection_objects_trgt_src= intersection_objects_trgt_src_without_mapping + intersection_objects_mapped_trgt_src
+        
+        return intersection_objects_trgt_src

icsDataValidation/utils/file_util.py

@@ -0,0 +1,96 @@
+import errno
+import os
+from typing import Union
+import json
+import decimal
+import numpy as np
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return str(o)
+        if isinstance(o, np.integer):
+            return int(o)
+        if isinstance(o, np.floating):
+            return float(o)
+        if isinstance(o, np.ndarray):
+            return o.tolist()
+        try:
+            super(CustomJSONEncoder, self).default(o)
+        except:
+            return str(o)
+
+        return super(CustomJSONEncoder, self).default(o)
+
+def load_file(file_path: str, encoding: str = "utf-8-sig") -> str:
+    """
+        Reads and returns the file content of given path.
+        Encodings tried in following order:
+                utf-8-sig - default
+                utf-8
+                utf-16-le
+                utf-16
+                cp1252
+    Args:
+        file_path: absolute file path to a file
+        encoding: specific code page name
+    Raises:
+        EnvironmentError - if file could not been read with stated encodings
+    Returns:
+        File content as string representation
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), file_path)
+    else:
+        try:
+            with open(file_path, "r", encoding=encoding) as file:
+                return file.read()
+        except:
+            try:
+                encoding = "utf-8"
+                with open(file_path, "r", encoding=encoding) as file:
+                    return file.read()
+            except:
+                try:
+                    encoding = "utf-16-le"
+                    with open(file_path, "r", encoding=encoding) as file:
+                        return file.read()
+                except:
+                    try:
+                        encoding = "utf-16"
+                        with open(file_path, "r", encoding=encoding) as file:
+                            return file.read()
+                    except:
+                        try:
+                            encoding = "cp1252"
+                            with open(file_path, "r", encoding=encoding) as file:
+                                return file.read()
+                        except:
+                            raise EnvironmentError(
+                                f"Can not read file {file_path}. Tried utf-8-sig (BOM), utf-8, utf-16, utf-16-le and cp1252."
+                            )
+
+def load_json(file_path: str, encoding: str = "utf-8-sig") -> Union[dict, None]:
+    """
+        Reads amd returns a given json file. Content must be in valid JSON Schema.
+        Valid JSON Schema should not have any trailing commas.
+        Encodings tried in following order:
+                utf-8-sig - default
+                utf-8
+                utf-16-le
+                utf-16
+                cp1252
+    Args:
+        file_path: absolute file path to a file
+        encoding: specific code page name
+    Raises:
+        EnvironmentError - if file could not been read with stated encodings
+    Returns:
+        File content as dictionary.
+        If the file path does not exists None will be returned.
+    """
+    return json.loads(load_file(file_path, encoding))
+
+
+def write_json_to_file(json_object: dict, file_path: str) -> None:
+    with open(f"{file_path}", 'w') as f:
+        json.dump(json_object, f, indent=4, cls=CustomJSONEncoder)

icsDataValidation/utils/logger_util.py

@@ -0,0 +1,96 @@
+import logging
+import re
+
+class DevOpsFormatter(logging.Formatter):
+    error_format  = "##vso[task.logissue type=error][%(name)s]   %(message)s"
+    warning_format = "##vso[task.logissue type=warning][%(name)s]   %(message)s"
+    dbg_fmt  = "DBG: %(module)s: %(lineno)d: %(msg)s"
+    info_format = "%(message)s"
+    section_format = "%(message)s"
+
+    def __init__(self, fmt="%(levelno)s: %(msg)s", section_info = False):
+        super().__init__(fmt=fmt)
+        self._section_info = section_info
+
+    def parse_progress(self, message:str) -> str:
+        """
+            Parses Method for progress information
+        """
+        progress_pattern = re.compile(r".*PROGRESS\s\[\s*'(?P<x>\d+)\/(?P<y>\d+)'\s*\].*", re.IGNORECASE | re.MULTILINE)
+
+        if (progress_pattern.match(message)):
+            progress_match = progress_pattern.search(message)
+            x = int(progress_match.group('x'))
+            y = int(progress_match.group('y'))
+            progress_value = round((x / y)*100)
+            return  f"##vso[task.setprogress value={progress_value};]script progress\n"
+
+        return ""
+
+    def parse_group_start(self, message:str) -> str:
+        """
+            Parses if its a group start and prepends a command string to the message
+        """
+        start_pattern = re.compile(r".*START\s(?P<gname>.+?\s\[\s*'.*'\s*\]).*", re.IGNORECASE | re.MULTILINE)
+
+        if (start_pattern.match(message)):
+            start_match = start_pattern.search(message)
+            return f"##[group]{start_match.group('gname')}\n"
+
+        return ""
+
+    def parse_group_end(self, message:str) -> str:
+        """
+            Parses if its a group end and appends a command string to the message
+        """
+        end_pattern = re.compile(r".*FINISHED\s.+?\s\[\s*'.*'\s*\].*", re.IGNORECASE | re.MULTILINE)
+
+        if (end_pattern.match(message)):
+            return "\n##[endgroup]"
+
+        return ""
+
+    def format(self, record):
+
+        # Save the original format configured by the user
+        # when the logger formatter was instantiated
+        format_orig = self._style._fmt
+
+        if (record.levelno == logging.INFO):
+            record_message = f"{record.msg}"
+
+            return f"{self.parse_progress(record.msg)}{self.parse_group_start(record.msg)}{record_message}{self.parse_group_end(record.msg)}"
+
+        # Replace the original format with one customized by logging level
+        if record.levelno == logging.DEBUG:
+            self._fmt = DevOpsFormatter.dbg_fmt
+        elif record.levelno == logging.INFO and not self._section_info:
+            self._style._fmt = DevOpsFormatter.info_format
+
+        elif record.levelno == logging.INFO and self._section_info:
+            self._style._fmt = DevOpsFormatter.section_format
+
+        elif record.levelno == logging.ERROR:
+            self._style._fmt = DevOpsFormatter.error_format
+
+        elif record.levelno == logging.WARNING:
+            self._style._fmt = DevOpsFormatter.warning_format
+
+        # Call the original formatter class to do the grunt work
+        result = logging.Formatter.format(self, record)
+
+        # Restore the original format configured by the user
+        self._style._fmt = format_orig
+
+        return result
+
+def configure_dev_ops_logger(logger: logging.Logger) -> None:
+    """
+        Configure logging for azure devops
+    """
+    snf_logger = logging.getLogger('snowflake.connector.ocsp_snowflake')
+    snf_logger.disabled = True
+    section_formatter = DevOpsFormatter(section_info=True)
+    section_handler = logging.StreamHandler()
+    section_handler.setFormatter(section_formatter)
+    logger.addHandler(section_handler)

icsDataValidation/utils/pandas_util.py

@@ -0,0 +1,159 @@
+import numpy as np
+from decimal import Decimal
+
+
+
+def get_diff_dataframes(df_1, df_2, key_columns_1, key_columns_2):
+    """
+    Get the the difference between two Pandas Dataframes by sorting over specific key-columns.
+    Returns the two dataframes containing only the rows with differences from the input dataframes and the sorted dataframes.
+    """
+    df_1_sorted = df_1.sort_values(by=key_columns_1).reset_index(drop=True)
+    df_2_sorted = df_2.sort_values(by=key_columns_2).reset_index(drop=True)
+
+    diff_1 = df_1_sorted[~df_1_sorted.apply(tuple,1).isin(df_2_sorted.apply(tuple,1))]
+    diff_2 = df_2_sorted[~df_2_sorted.apply(tuple,1).isin(df_1_sorted.apply(tuple,1))]
+
+    diff_1 = diff_1.reset_index(drop=True)
+    diff_2 = diff_2.reset_index(drop=True)
+
+    return diff_1, diff_2, df_1_sorted, df_2_sorted
+
+
+def get_diff_dict_from_diff_dataframes(diff_1, diff_2, key_columns_1, key_columns_2, key_column_values_with_mismatches, numeric_scale):
+    """
+    Get the 
+    """
+    diff_dict = {}
+
+    #TODO support a list of key_columns_1 (and key_columns_2) and a dictionary of key_column_values_with_mismatches
+    key_column_1=key_columns_1[0]
+    key_column_2=key_columns_2[0]
+    key_column_values_with_mismatches=key_column_values_with_mismatches[key_column_1]
+
+    for value_with_mismatch in key_column_values_with_mismatches:
+        
+        if value_with_mismatch is None:
+            row_1_with_mismatch = diff_1.loc[diff_1[key_column_1].isnull()]
+            row_2_with_mismatch = diff_2.loc[diff_2[key_column_2].isnull()]
+            value_with_mismatch = 'NULL'
+        else:
+            row_1_with_mismatch = diff_1.loc[diff_1[key_column_1] == value_with_mismatch]
+            row_2_with_mismatch = diff_2.loc[diff_2[key_column_2] == value_with_mismatch]
+            value_with_mismatch = str(value_with_mismatch)
+
+        diff_dict[value_with_mismatch] = {}
+
+        for column in row_1_with_mismatch:
+            if column == 'group_by_column' or column not in row_2_with_mismatch or column in key_columns_1:
+                continue
+
+            if row_1_with_mismatch[column].values.size > 0:
+                src_value=row_1_with_mismatch[column].values[0]
+            elif column=='COUNT_OF_GROUP_BY_VALUE':
+                src_value=0
+            else:
+                src_value= None
+
+            if row_2_with_mismatch[column].values.size > 0:
+                trgt_value=row_2_with_mismatch[column].values[0]
+            elif column=='COUNT_OF_GROUP_BY_VALUE':
+                trgt_value=0
+            else:
+                trgt_value= None
+            
+            try:
+                src_value= src_value.item()
+            except Exception:
+                pass
+
+            try:
+                trgt_value= trgt_value.item()
+            except Exception:
+                pass
+            
+            if src_value != trgt_value: 
+                if src_value is None:
+                    diff_trgt_minus_src = trgt_value
+                elif trgt_value is None:
+                    if isinstance(src_value, str) or isinstance(trgt_value, str):
+                        diff_trgt_minus_src = f"{-int(src_value.split('_',1)[0])}_{-int(src_value.split('_',1)[1])}"
+                    else:
+                        diff_trgt_minus_src = -round(float(src_value), numeric_scale)
+                else:
+                    if isinstance(src_value, str) or isinstance(trgt_value, str):
+                        diff_trgt_minus_src = f"{int(trgt_value.split('_',1)[0])-int(src_value.split('_',1)[0])}_{int(trgt_value.split('_',1)[1])-int(src_value.split('_',1)[1])}"
+                    else:
+                        diff_trgt_minus_src = round(float(trgt_value)-float(src_value), numeric_scale)
+                
+                if diff_trgt_minus_src:
+                    diff_dict[value_with_mismatch][column] = {
+                        "SRC_VALUE": src_value,
+                        "TRGT_VALUE": trgt_value,
+                        "DIFF_TRGT_MINUS_SRC": diff_trgt_minus_src
+                        }
+        if not diff_dict[value_with_mismatch]:
+            diff_dict.pop(value_with_mismatch)
+
+    if not diff_dict:
+        diff_dict = None
+
+    return diff_dict
+
+
+
+#########################################################################################################
+#TODO write as pytest
+# Test Space 
+import pandas as pd
+
+# *** TEST 1 ***
+#df1 = pd.DataFrame({'group_by_column': [1, 2,3,6], 'A': [1, 1,9,2], 'B': [1, 3,9,2], 'C': [1, 4,9,2],'D': ['1_1', '1_1','8_3','1_1']})
+#df2 = pd.DataFrame({'group_by_column': [2, 1,3,5], 'A': [2, 1,9,1], 'B': [3, 1,5,1], 'C': [5, 1,9,1],'D': ['1_1', '1_1','5_5','1_1']})
+#key_column_values_with_mismatches={'group_by_column':[None,3,5,6]}
+#numeric_scale=2
+
+#########
+
+# *** TEST 2 ***
+#df1 = pd.DataFrame({'group_by_column': [1, None,3,6], 'A': [1, 1,9,2], 'B': [1, 3,9,2], 'C': [1, 4,9,2],'D': ['1_1', '1_1','8_3','1_1']})
+#df2 = pd.DataFrame({'group_by_column': [None, 1,3,5], 'A': [2, 1,9,1], 'B': [3, 1,5,1], 'C': [5, 1,9,1],'D': ['1_1', '1_1','5_5','1_1']})
+#key_column_values_with_mismatches={'group_by_column':[None,3,5,6]}
+#numeric_scale=2
+
+#########
+
+# *** TEST 3 ***
+#df1 = pd.DataFrame({'group_by_column': [1, 2,3,4], 'A': [1, 1,9,2.001], 'B': [1, 3,9,2.0004], 'C': [1, 4,9,2.00000000001],'D': ['1_1', '1_1','8_3','1_1']})
+#df2 = pd.DataFrame({'group_by_column': [1, 2,3,4], 'A': [2, 1,9,Decimal(2.001)], 'B': [3, 3,9,2.0005], 'C': [1, 4,9,2.00000000004],'D': ['1_1', '1_1','8_3','1_1']})
+#key_column_values_with_mismatches={'group_by_column':[1,2,3,4]}
+#numeric_scale=7
+#
+##########
+#
+#diff_1, diff_2, df_1_sorted, df_2_sorted =get_diff_dataframes(df1, df2, ['group_by_column'], ['group_by_column'])
+#diff_dict = get_diff_dict_from_diff_dataframes(df1, df2, ['group_by_column'], ['group_by_column'], key_column_values_with_mismatches, numeric_scale)
+#import json
+#import decimal
+#import numpy as np
+#
+#class CustomJSONEncoder(json.JSONEncoder):
+#    def default(self, o):
+#        if isinstance(o, decimal.Decimal):
+#            return str(o)
+#        if isinstance(o, np.integer):
+#            return int(o)
+#        if isinstance(o, np.floating):
+#            return float(o)
+#        if isinstance(o, np.ndarray):
+#            return o.tolist()
+#        try:
+#            super(CustomJSONEncoder, self).default(o)
+#        except:
+#            return str(o)
+#
+#        return super(CustomJSONEncoder, self).default(o)
+#
+#diff_json = json.dumps(diff_dict, indent=4, cls=CustomJSONEncoder)
+#
+#print(diff_json)