icsDataValidation 1.0.271__py3-none-any.whl → 1.0.273__py3-none-any.whl

icsDataValidation/core/object_comparison.py

@@ -122,7 +122,7 @@ def compare_objects(testing_tool_params: TestingToolParams, objects_to_compare:
 
             for column in comparison_service.result_params.all_columns_trgt_src:
 
-                column_level_comparison_result=ResultService.prepare_column_level_result(column, exclude_columns, comparison_service.result_params, testing_tool_params)
+                column_level_comparison_result=ResultService.prepare_column_level_result(column, exclude_columns, comparison_service.result_params)
 
                 if column_level_comparison_result["COUNT_NULLS_EQUAL"] is False:
                     comparison_service.result_params.all_count_nulls_equal = False

icsDataValidation/output_parameters/result_params.py

@@ -49,7 +49,6 @@ class ResultParams():
         # aggregation-comparison (to save)
         aggregations_equal = None
         all_count_nulls_equal = None
-        aggregations_equal_tolerated = None
 
         # error handling (row-count-comparison and aggregation-comparison)
         src_error_dict = None

icsDataValidation/services/comparison_service.py

@@ -4,6 +4,7 @@ import datetime
 import numpy as np
 
 from pandas._testing import assert_frame_equal
+from decimal import Decimal
 
 from icsDataValidation.utils.logger_util import configure_dev_ops_logger
 from icsDataValidation.utils.pandas_util import get_diff_dataframes, get_diff_dict_from_diff_dataframes
@@ -166,8 +167,8 @@ class ComparisonService(TestingToolParams):
 
         if self.result_params.src_row_count != 0 and self.result_params.trgt_row_count != 0:
             aggregation_differences_trgt_minus_src_not_boolean = {
-                                                                    k:  round(float(trgt_columns_aggregate[k][1]) 
-                                                                        - float(src_columns_aggregate[k][1]), self.numeric_scale) 
+                                                                    k:  round(Decimal(trgt_columns_aggregate[k][1]) 
+                                                                        - Decimal(src_columns_aggregate[k][1]), self.numeric_scale) 
                                                                     for k in src_columns_aggregate.keys() 
                                                                         if k in trgt_columns_aggregate 
                                                                         and str(src_columns_aggregate[k][1]) != str(trgt_columns_aggregate[k][1]) 
@@ -240,20 +241,6 @@ class ComparisonService(TestingToolParams):
                 if aggregation_diff and not aggregation_diff == 0.0:
                     aggregations_equal = False
                     break 
-        aggregations_equal_tolerated = aggregations_equal
-
-        if 'DATATYPE_TOLERACE' not in self.migration_config['MAPPING'].keys():
-            aggregations_equal_tolerated = None
-        else:
-            numeric_cols = {i['COLUMN_NAME']:i['DATA_TYPE'] for i in src_column_datatypes if i['DATA_TYPE'] in self.migration_config['MAPPING']['DATATYPE_TOLERACE'].keys()}
-            for col in numeric_cols.keys():
-                if (
-                    col in aggregation_differences_trgt_minus_src.keys() 
-                    and  src_columns_aggregate[col][0]=='SUM' 
-                    and abs(aggregation_differences_trgt_minus_src[col]) <= self.migration_config["MAPPING"]["DATATYPE_TOLERACE"][numeric_cols[col]]
-                    ):
-                    aggregations_equal_tolerated = True
-                     
 
         # save results              
         self.result_params.src_column_datatypes = src_column_datatypes
@@ -266,8 +253,6 @@ class ComparisonService(TestingToolParams):
         self.result_params.src_error_dict = src_error_dict
         self.result_params.trgt_error_dict = trgt_error_dict
         self.result_params.aggregations_equal = aggregations_equal 
-        self.result_params.aggregations_equal_tolerated = aggregations_equal_tolerated
-
 
 
     def group_by_comparison(self):

icsDataValidation/services/result_service.py

@@ -4,6 +4,7 @@ import subprocess
 import json
 
 from azure.storage.blob import BlobServiceClient
+from decimal import Decimal
 
 from icsDataValidation.services.system_service import SystemService
 from icsDataValidation.utils.logger_util import configure_dev_ops_logger
@@ -107,8 +108,7 @@ class ResultService(TestingToolParams):
     def prepare_column_level_result(
             column: str, 
             exclude_columns: list, 
-            result_params: ResultParams,
-            testing_tool_params: TestingToolParams
+            result_params: ResultParams
         ) -> dict:
         """
         Get column level result dictionary from the result parameters.
@@ -187,7 +187,7 @@ class ResultService(TestingToolParams):
                 if (
                     src_datatype in TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERACE'].keys() 
                     and aggregation_type == 'SUM' 
-                    and abs(float(aggregation_difference_trgt_minus_src)) <= TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERACE'][src_datatype]
+                    and abs(Decimal(aggregation_difference_trgt_minus_src)) <= Decimal(TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERACE'][src_datatype])
                     ):
                     aggregation_tolerated = True
             else : 
@@ -224,7 +224,7 @@ class ResultService(TestingToolParams):
         }
 
         return column_comparison_result
-
+    
     @staticmethod
     def prepare_object_level_result(
             src_object: DatabaseObject, 
@@ -238,6 +238,16 @@ class ResultService(TestingToolParams):
         """
         Get object level result dictionary from the result parameters and from the column level result.
         """
+
+        if 'DATATYPE_TOLERACE' not in TestingToolParams.migration_config['MAPPING']:
+            aggregations_equal_tolerated = None
+        elif result_params.aggregations_equal:
+            aggregations_equal_tolerated = True
+        elif all([column['AGGREGATION_EQUAL_TOLERATED'] for column in column_level_comparison_result]):
+            aggregations_equal_tolerated = True
+        else:
+            aggregations_equal_tolerated = False
+
         object_level_comparison_result = {
             "SRC_DATABASE_NAME": src_object.database,
             "SRC_SCHEMA_NAME": src_object.schema,
@@ -260,7 +270,7 @@ class ResultService(TestingToolParams):
             "TRGT_ROW_COUNT": result_params.trgt_row_count,
             "ALL_COUNT_NULLS_EQUAL": result_params.all_count_nulls_equal,   
             "AGGREGATIONS_EQUAL": result_params.aggregations_equal,  
-            "AGGREGATIONS_EQUAL_TOLERATED":result_params.aggregations_equal_tolerated,    
+            "AGGREGATIONS_EQUAL_TOLERATED": aggregations_equal_tolerated,    
             "SRC_ERROR": result_params.src_error_dict,
             "TRGT_ERROR":  result_params.trgt_error_dict,
             "GROUP_BY_COLUMNS": result_params.object_group_by_columns,

{icsDataValidation-1.0.271.dist-info → icsDataValidation-1.0.273.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: icsDataValidation
-Version: 1.0.271
+Version: 1.0.273
 Summary: ics data validation
 Home-page: https://initions.com/
 Author: initions

{icsDataValidation-1.0.271.dist-info → icsDataValidation-1.0.273.dist-info}/RECORD

@@ -11,15 +11,15 @@ icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=JDTdIM0
 icsDataValidation/connection_setups/teradata_connection_setup.py,sha256=fIpuxz-FTqFK2vSMSuokqU9sdJkaJ4UP5piY_zIbj5k,624
 icsDataValidation/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/core/database_objects.py,sha256=2oaDaVQajSYI_HJjJy1pmc6FsoK_wMfwgu6ZgEcFvow,523
-icsDataValidation/core/object_comparison.py,sha256=yOY_k0KTGtRqCCLBlveTtVMm0J-oape1Y-U8fUe4eU0,15009
+icsDataValidation/core/object_comparison.py,sha256=OEz5m1pp_PbIWyM5998iB5obFKYdJEqDo9Z0Hpj7o4A,14988
 icsDataValidation/input_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/input_parameters/testing_tool_params.py,sha256=6LkqEaH3vaeCn6d1b3jYIwlXAXIRclxLzcXdsmlkc9M,6731
 icsDataValidation/output_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/output_parameters/result_params.py,sha256=fgWI7Qm2rCOD7qWidgMAtS-t0r1olAa6HQxxuQDVEA0,2877
+icsDataValidation/output_parameters/result_params.py,sha256=5Mk9L9zWaxUqcKwLZQ539lVUp0b0s-YUmSA3PBgbqfs,2833
 icsDataValidation/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/services/comparison_service.py,sha256=X-Qvkeqn1wuQ7qiqXYfLSSbiXBS-Moe3JzE2dStYSBo,43586
+icsDataValidation/services/comparison_service.py,sha256=x8RjZs2bgYDHFueq5ysNADY1cC6rYBf6eDyCsF-_w84,42705
 icsDataValidation/services/initialization_service.py,sha256=AHbJrq_LjMPFoeOJC2pi2ZZ1xkL8njSZn38psc3do60,6687
-icsDataValidation/services/result_service.py,sha256=RAp8b6cuCAmN9iwfcASTebg7Dx4ngo46EnTP8zSJC5o,27916
+icsDataValidation/services/result_service.py,sha256=XwesiZjs8nAFD4wTYM_Fd1xEH_Txug3mj1FzpAg7o7c,28339
 icsDataValidation/services/system_service.py,sha256=GSkSPNG5PlLWchwlYM5H-1FMtuCNwpXcyZZOUB_0stU,3228
 icsDataValidation/services/testset_service.py,sha256=k1wRjI4Ltw9fylek9iW8N6DvnXn13wf6IJ703qQDMEc,15363
 icsDataValidation/services/database_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,7 +36,7 @@ icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI
 icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
 icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
 icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
-icsDataValidation-1.0.271.dist-info/METADATA,sha256=1wnvWz7FIP94p-JVGTbz4z8oWbE9XmgpXurJSdNNCb8,720
-icsDataValidation-1.0.271.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-icsDataValidation-1.0.271.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
-icsDataValidation-1.0.271.dist-info/RECORD,,
+icsDataValidation-1.0.273.dist-info/METADATA,sha256=Ya2fksiXg3MA7a8LkygIvQuTjOCkFFSIch9fJ_5fKGI,720
+icsDataValidation-1.0.273.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+icsDataValidation-1.0.273.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
+icsDataValidation-1.0.273.dist-info/RECORD,,