icsDataValidation 1.0.252__py3-none-any.whl → 1.0.271__py3-none-any.whl

icsDataValidation/output_parameters/result_params.py

@@ -49,6 +49,7 @@ class ResultParams():
         # aggregation-comparison (to save)
         aggregations_equal = None
         all_count_nulls_equal = None
+        aggregations_equal_tolerated = None
 
         # error handling (row-count-comparison and aggregation-comparison)
         src_error_dict = None

icsDataValidation/services/comparison_service.py

@@ -231,7 +231,7 @@ class ComparisonService(TestingToolParams):
 
         else:
             aggregation_differences_trgt_minus_src = {}
-
+                                                     
         aggregations_equal = True
         if src_aggregations_error or trgt_aggregations_error:
             aggregations_equal = None
@@ -240,8 +240,22 @@ class ComparisonService(TestingToolParams):
                 if aggregation_diff and not aggregation_diff == 0.0:
                     aggregations_equal = False
                     break 
+        aggregations_equal_tolerated = aggregations_equal
 
-        # save results
+        if 'DATATYPE_TOLERACE' not in self.migration_config['MAPPING'].keys():
+            aggregations_equal_tolerated = None
+        else:
+            numeric_cols = {i['COLUMN_NAME']:i['DATA_TYPE'] for i in src_column_datatypes if i['DATA_TYPE'] in self.migration_config['MAPPING']['DATATYPE_TOLERACE'].keys()}
+            for col in numeric_cols.keys():
+                if (
+                    col in aggregation_differences_trgt_minus_src.keys() 
+                    and  src_columns_aggregate[col][0]=='SUM' 
+                    and abs(aggregation_differences_trgt_minus_src[col]) <= self.migration_config["MAPPING"]["DATATYPE_TOLERACE"][numeric_cols[col]]
+                    ):
+                    aggregations_equal_tolerated = True
+                     
+
+        # save results              
         self.result_params.src_column_datatypes = src_column_datatypes
         self.result_params.src_columns_aggregate = src_columns_aggregate
         self.result_params.trgt_column_datatypes = trgt_column_datatypes
@@ -249,10 +263,11 @@ class ComparisonService(TestingToolParams):
         self.result_params.src_aggregations_error = src_aggregations_error
         self.result_params.trgt_aggregations_error = trgt_aggregations_error
         self.result_params.aggregation_differences_trgt_minus_src  = aggregation_differences_trgt_minus_src
-
         self.result_params.src_error_dict = src_error_dict
         self.result_params.trgt_error_dict = trgt_error_dict
-        self.result_params.aggregations_equal = aggregations_equal    
+        self.result_params.aggregations_equal = aggregations_equal 
+        self.result_params.aggregations_equal_tolerated = aggregations_equal_tolerated
+
 
 
     def group_by_comparison(self):

icsDataValidation/services/database_services/snowflake_service.py

@@ -852,6 +852,7 @@ class SnowflakeService(object):
                                                                         TRGT_ROW_COUNT, \
                                                                         ALL_COUNT_NULLS_EQUAL, \
                                                                         AGGREGATIONS_EQUAL, \
+                                                                        AGGREGATIONS_EQUAL_TOLERATED,\
                                                                         SRC_ERROR_QUERY, \
                                                                         TRGT_ERROR_QUERY, \
                                                                         SRC_ERROR_MSG, \
@@ -906,6 +907,7 @@ class SnowflakeService(object):
                                                 F1.VALUE:TRGT_ROW_COUNT::INT AS TRGT_ROW_COUNT, \
                                                 F1.VALUE:ALL_COUNT_NULLS_EQUAL::BOOLEAN AS ALL_COUNT_NULLS_EQUAL, \
                                                 F1.VALUE:AGGREGATIONS_EQUAL::BOOLEAN AS AGGREGATIONS_EQUAL, \
+                                                F1.VALUE:AGGREGATIONS_EQUAL_TOLERATED::BOOLEAN AS AGGREGATIONS_EQUAL_TOLERATED,\
                                                 F1.VALUE:SRC_ERROR:QUERY::VARCHAR AS SRC_ERROR_QUERY, \
                                                 F1.VALUE:TRGT_ERROR:QUERY::VARCHAR AS TRGT_ERROR_QUERY, \
                                                 F1.VALUE:SRC_ERROR:ERROR::VARCHAR AS SRC_ERROR_MSG, \
@@ -986,6 +988,7 @@ class SnowflakeService(object):
                                                                 AGGREGATION_RESULT_SRC,\
                                                                 AGGREGATION_RESULT_TRGT,\
                                                                 AGGREGATION_DIFFERENCE_TRGT_MINUS_SRC,\
+                                                                AGGREGATION_EQUAL_TOLERATED,\
                                                                 COUNT_NULLS_EQUAL,\
                                                                 COUNT_NULLS_SRC,\
                                                                 COUNT_NULLS_TRGT,\
@@ -1021,6 +1024,7 @@ class SnowflakeService(object):
                                                             F2.VALUE:AGGREGATION_RESULT_SRC::VARCHAR AS AGGREGATION_RESULT_SRC,\
                                                             F2.VALUE:AGGREGATION_RESULT_TRGT::VARCHAR AS AGGREGATION_RESULT_TRGT,\
                                                             F2.VALUE:AGGREGATION_DIFFERENCE_TRGT_MINUS_SRC::VARCHAR AS AGGREGATION_DIFFERENCE_TRGT_MINUS_SRC,\
+                                                            F2.VALUE:AGGREGATION_EQUAL_TOLERATED::BOOLEAN AS AGGREGATION_EQUAL_TOLERATED,\
                                                             F2.VALUE:COUNT_NULLS_EQUAL::BOOLEAN AS COUNT_NULLS_EQUAL,\
                                                             F2.VALUE:COUNT_NULLS_SRC::VARCHAR AS COUNT_NULLS_SRC,\
                                                             F2.VALUE:COUNT_NULLS_TRGT::VARCHAR AS COUNT_NULLS_TRGT,\

icsDataValidation/services/result_service.py

@@ -121,6 +121,7 @@ class ResultService(TestingToolParams):
         aggregation_result_src = None
         aggregation_result_trgt = None
         aggregation_equal = None
+        aggregation_tolerated = None
         aggregation_difference_trgt_minus_src = None                   
         count_nulls_src = None
         count_nulls_trgt = None
@@ -133,7 +134,7 @@ class ResultService(TestingToolParams):
         else:
             in_src = False
             src_datatype = None
-        
+
         if column in result_params.trgt_columns_upper:
             in_trgt = True
             trgt_datatype = next(item["DATA_TYPE"] for item in result_params.trgt_column_datatypes if item["COLUMN_NAME"].upper() == column)
@@ -163,15 +164,34 @@ class ResultService(TestingToolParams):
                 if column in result_params.aggregation_differences_trgt_minus_src and result_params.aggregation_differences_trgt_minus_src[column] and not result_params.aggregation_differences_trgt_minus_src[column] == '0_0':
                     aggregation_equal = False
                     aggregation_difference_trgt_minus_src = result_params.aggregation_differences_trgt_minus_src[column]
+                    
                 elif aggregation_result_src is not None and aggregation_result_trgt is not None and aggregation_type_src and aggregation_type_trgt and aggregation_type_src == aggregation_type_trgt:
                     aggregation_equal = True
+                        
                     if column in result_params.aggregation_differences_trgt_minus_src and result_params.aggregation_differences_trgt_minus_src[column] == '0_0':
                         aggregation_difference_trgt_minus_src='0_0'
-                    else:
+                        
+                    else: 
                         aggregation_difference_trgt_minus_src='0'
-
+                                    
                 if aggregation_type_src and aggregation_type_trgt and aggregation_type_src == aggregation_type_trgt:
                     aggregation_type = aggregation_type_src
+            
+            '''
+            Comparison Based on Decimal Places
+            Logic is defined in migration_config.json
+            '''
+            aggregation_tolerated = aggregation_equal
+        
+            if 'DATATYPE_TOLERACE' in TestingToolParams.migration_config['MAPPING'].keys():
+                if (
+                    src_datatype in TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERACE'].keys() 
+                    and aggregation_type == 'SUM' 
+                    and abs(float(aggregation_difference_trgt_minus_src)) <= TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERACE'][src_datatype]
+                    ):
+                    aggregation_tolerated = True
+            else : 
+                aggregation_tolerated = None
                     
             if count_nulls_src is not None and count_nulls_trgt is not None and count_nulls_src==count_nulls_trgt:
                 count_nulls_equal = True
@@ -179,7 +199,7 @@ class ResultService(TestingToolParams):
             elif count_nulls_src is not None and count_nulls_trgt is not None:
                     count_nulls_equal = False
                     count_nulls_difference_trgt_minus_src = int(count_nulls_trgt)-int(count_nulls_src)
-
+                
             datatype_equal = ResultService._compare_column_datatypes(src_datatype, trgt_datatype)
 
         column_comparison_result = {
@@ -193,6 +213,7 @@ class ResultService(TestingToolParams):
             "DATATYPE_EQUAL": datatype_equal,
             "AGGREGATION_TYPE": aggregation_type,
             "AGGREGATION_EQUAL": aggregation_equal,
+            "AGGREGATION_EQUAL_TOLERATED": aggregation_tolerated,
             "AGGREGATION_RESULT_SRC": aggregation_result_src,
             "AGGREGATION_RESULT_TRGT": aggregation_result_trgt,
             "AGGREGATION_DIFFERENCE_TRGT_MINUS_SRC": aggregation_difference_trgt_minus_src,
@@ -238,7 +259,8 @@ class ResultService(TestingToolParams):
             "SRC_ROW_COUNT": result_params.src_row_count,
             "TRGT_ROW_COUNT": result_params.trgt_row_count,
             "ALL_COUNT_NULLS_EQUAL": result_params.all_count_nulls_equal,   
-            "AGGREGATIONS_EQUAL": result_params.aggregations_equal,         
+            "AGGREGATIONS_EQUAL": result_params.aggregations_equal,  
+            "AGGREGATIONS_EQUAL_TOLERATED":result_params.aggregations_equal_tolerated,    
             "SRC_ERROR": result_params.src_error_dict,
             "TRGT_ERROR":  result_params.trgt_error_dict,
             "GROUP_BY_COLUMNS": result_params.object_group_by_columns,

{icsDataValidation-1.0.252.dist-info → icsDataValidation-1.0.271.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: icsDataValidation
-Version: 1.0.252
+Version: 1.0.271
 Summary: ics data validation
 Home-page: https://initions.com/
 Author: initions

{icsDataValidation-1.0.252.dist-info → icsDataValidation-1.0.271.dist-info}/RECORD

@@ -15,11 +15,11 @@ icsDataValidation/core/object_comparison.py,sha256=yOY_k0KTGtRqCCLBlveTtVMm0J-oa
 icsDataValidation/input_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/input_parameters/testing_tool_params.py,sha256=6LkqEaH3vaeCn6d1b3jYIwlXAXIRclxLzcXdsmlkc9M,6731
 icsDataValidation/output_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/output_parameters/result_params.py,sha256=5Mk9L9zWaxUqcKwLZQ539lVUp0b0s-YUmSA3PBgbqfs,2833
+icsDataValidation/output_parameters/result_params.py,sha256=fgWI7Qm2rCOD7qWidgMAtS-t0r1olAa6HQxxuQDVEA0,2877
 icsDataValidation/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/services/comparison_service.py,sha256=0Ki7TvIiYL-CS_DqlO-aV83JNm4GoQNbG7JgnPbMoB4,42610
+icsDataValidation/services/comparison_service.py,sha256=X-Qvkeqn1wuQ7qiqXYfLSSbiXBS-Moe3JzE2dStYSBo,43586
 icsDataValidation/services/initialization_service.py,sha256=AHbJrq_LjMPFoeOJC2pi2ZZ1xkL8njSZn38psc3do60,6687
-icsDataValidation/services/result_service.py,sha256=oYUAGkxjCDK0CaAxhJRBj8q1brs60aKLUJAV8a-MXF8,26834
+icsDataValidation/services/result_service.py,sha256=RAp8b6cuCAmN9iwfcASTebg7Dx4ngo46EnTP8zSJC5o,27916
 icsDataValidation/services/system_service.py,sha256=GSkSPNG5PlLWchwlYM5H-1FMtuCNwpXcyZZOUB_0stU,3228
 icsDataValidation/services/testset_service.py,sha256=k1wRjI4Ltw9fylek9iW8N6DvnXn13wf6IJ703qQDMEc,15363
 icsDataValidation/services/database_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,7 +28,7 @@ icsDataValidation/services/database_services/databricks_hive_metastore_service.p
 icsDataValidation/services/database_services/databricks_unity_catalog_service.py,sha256=INA8rd3KW_jAplNagGa9tEON3dyOufcIAPOOdmc0Mrc,70259
 icsDataValidation/services/database_services/exasol_service.py,sha256=7LYnRScO3DxBmuSN0HmTgsFc2el-Ii3A9jgGsXSJVU8,11074
 icsDataValidation/services/database_services/oracle_service.py,sha256=60unwWlHm520ioFmz0y2K8ApwZrruf9iB0ojjQx0IWc,31523
-icsDataValidation/services/database_services/snowflake_service.py,sha256=Z9D-Y5B7hV3HpxU1ZpQDJNCu05TeB5ZXl5Fj36r3ptM,60165
+icsDataValidation/services/database_services/snowflake_service.py,sha256=UWmjQZN4oX2ctH6uhE2oklXGHo66SK5UnQbFYFhzDuc,60630
 icsDataValidation/services/database_services/teradata_service.py,sha256=Rf0xzcZGEbooq3r2Rfe2fCahTm2Xw4uznQa8vyWoyqM,40169
 icsDataValidation/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/utils/file_util.py,sha256=ZTMB1sTnIIdffg9tEJRCFQQ5SG8Fksc5ie1PM4gHXG4,3432
@@ -36,7 +36,7 @@ icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI
 icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
 icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
 icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
-icsDataValidation-1.0.252.dist-info/METADATA,sha256=sEdDCNO4CPlvGgJL5-CK8JF2LwwB3qEF_rypgsOkgwg,720
-icsDataValidation-1.0.252.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-icsDataValidation-1.0.252.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
-icsDataValidation-1.0.252.dist-info/RECORD,,
+icsDataValidation-1.0.271.dist-info/METADATA,sha256=1wnvWz7FIP94p-JVGTbz4z8oWbE9XmgpXurJSdNNCb8,720
+icsDataValidation-1.0.271.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+icsDataValidation-1.0.271.dist-info/top_level.txt,sha256=YL9V1qreCXZeUCy-tzA4Vxv5-6mvXy5lsfAT0nQapfg,53
+icsDataValidation-1.0.271.dist-info/RECORD,,