icsDataValidation 1.0.425__tar.gz → 1.0.428__tar.gz

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: icsDataValidation
-Version: 1.0.425
+Version: 1.0.428
 Summary: Add your description here
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/comparison_service.py

@@ -323,9 +323,8 @@ class ComparisonService(TestingToolParams):
         elif self.use_group_by_columns and self.migration_config["GROUP_BY_AGGREGATION"]["GROUP_BY_COLUMNS"]:
             logger.info(f"[{self.comp_id}] START Group-By-Comparison - with option 2 (group_by_columns defined as a list for all objects)")
             global_group_by_columns=self.migration_config["GROUP_BY_AGGREGATION"]["GROUP_BY_COLUMNS"]
-            global_group_by_columns_in_object=[group_by_column for group_by_column in global_group_by_columns if group_by_column in self.result_params.intersection_columns_trgt_src]
-            if global_group_by_columns_in_object:
-                object_group_by_columns=[global_group_by_columns_in_object]
+            object_group_by_columns=[group_by_column for group_by_column in global_group_by_columns if group_by_column in self.result_params.intersection_columns_trgt_src]
+            if object_group_by_columns:
                 object_group_by_aggregation_columns=["all"]
                 object_group_by_aggregation_type='various'
 

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/database_services/azure_service.py

@@ -309,10 +309,10 @@ class AzureService:
 
         try:
             for single_query in query_list:
+                query_result=self.azure_connection.execute(single_query)
+
                 if return_as_pdf:
-                    query_result = pandas.io.sql.read_sql(single_query, self.azure_connection)
-                else:
-                    query_result=self.azure_connection.execute(single_query)
+                    query_result = pd.DataFrame(query_result)
 
                 results.append(query_result)
 

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/database_services/snowflake_service.py

@@ -556,7 +556,7 @@ class SnowflakeService:
                     aggregates += f', COUNT(DISTINCT LOWER(TRY_TO_NUMBER({column_identifier}::VARCHAR))) AS "COUNTDISTINCT_{column}"'
 
                 elif not only_numeric and column_datatype.lower() in self.snowflake_datatype_mapping["boolean"]:
-                    aggregates += f", MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column_identifier} = true)::VARCHAR || '_' || MAX(SELECT COUNT(*) FROM {object.database}.{object.schema}.{object.name} WHERE {column_identifier} = false) :: VARCHAR AS \"AGGREGATEBOOLEAN_{column}\""
+                    aggregates += f", COUNT(CASE WHEN {column_identifier} = true THEN 1 ELSE NULL END)::VARCHAR || '_' || COUNT(CASE WHEN {column_identifier} = false THEN 1 ELSE NULL END)::VARCHAR AS \"AGGREGATEBOOLEAN_{column}\""
 
                 # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/database_services/sqlserver_service.py

@@ -81,21 +81,21 @@ class SQLServerService:
             str: in clause as string
         """
         values = list(key_filters.values())
-        in_clause_values = "('"
+        in_clause_values = "'"
         for j in range(len(values[0])):
             for value in values:
                 in_clause_values += str(value[j]) + "','"
-            in_clause_values = in_clause_values[:-2] + "),('"
-        in_clause_values = in_clause_values[:-3] + ")"
+            in_clause_values = in_clause_values[:-2] + ",'"
+        in_clause_values = in_clause_values[:-3] + "'"
 
         in_clause_cols = " AND (("
         for key in key_filters.keys():
             if key in numeric_columns:
-                in_clause_cols += f"""ROUND({key.replace("'", "")}, {numeric_scale})""" + ","
+                in_clause_cols += f"""cast(ROUND({key.replace("'", "")}, {numeric_scale}) as numeric(38, {numeric_scale}))""" + ","
             else:
                 in_clause_cols += key.replace("'", "") + ","
         in_clause_cols = in_clause_cols[:-1] + ")"
-        in_clause = in_clause_cols + " in (" + in_clause_values + ")"
+        in_clause = in_clause_cols + " in (" + in_clause_values + "))"
         return in_clause
 
     def _get_column_clause(self, column_list: list, columns_datatype: list, numeric_scale, key_columns,
@@ -484,7 +484,7 @@ class SQLServerService:
                 aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,{column}))) AS COUNTDISTINCT_{column}"
 
             elif column_datatype.lower() in self.sqlserver_datatype_mapping["boolean"]:
-                aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END)))) AS AGGREGATEBOOLEAN_{column}"
+                aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END))) AS AGGREGATEBOOLEAN_{column}"
 
             #else: Additional Data Types: image , sql_variant, uniqueidentifier, xml, cursor, table, column_datatype.lower() == 'bit' or
 
@@ -614,7 +614,7 @@ class SQLServerService:
                     aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,{column}))) AS COUNTDISTINCT_{column}"
 
                 elif not only_numeric and column_datatype.lower() in self.sqlserver_datatype_mapping["boolean"]:
-                    aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END)))) AS AGGREGATEBOOLEAN_{column}"
+                    aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END))) AS AGGREGATEBOOLEAN_{column}"
 
                 # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
@@ -744,7 +744,7 @@ class SQLServerService:
                             SELECT TOP ({sample_count}) {column_clause}
                                 FROM {object.schema}.{object.name}
                                 {where_clause}{in_clause}
-                                ORDER BY NEWID(), {keys};
+                                ORDER BY {keys};
                             """
         elif key_intersection != [] and not is_dedicated:
             keys = str(key_intersection)[1:-1].replace("'", "")
@@ -764,7 +764,7 @@ class SQLServerService:
                             SELECT TOP ({sample_count}) {column_clause}
                                 FROM {object.schema}.{object.name}
                                 {where_clause}{in_clause}
-                                ORDER BY NEWID(), {keys};
+                                ORDER BY {keys};
                             """
         else:
             column_intersections = list(set(column_intersections) - set(exclude_columns))
@@ -834,12 +834,12 @@ class SQLServerService:
 
         for single_query in query_list:
             try:
+                query_result=cursor.execute(single_query).fetchall()
+                columns = [column[0] for column in cursor.description]
+                query_result = [dict(zip(columns, row)) for row in query_result]
+
                 if return_as_pdf:
-                    query_result = pandas.io.sql.read_sql(single_query, self.sqlserver_connection)
-                else:
-                    query_result=cursor.execute(single_query).fetchall()
-                    columns = [column[0] for column in cursor.description]
-                    query_result = [dict(zip(columns, row)) for row in query_result]
+                    query_result = pd.DataFrame(query_result)
 
                 results.append(query_result)
             except Exception as err:

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/database_services/teradata_service.py

@@ -641,11 +641,11 @@ class TeradataService(object):
 
         for single_query in query_list:
             try:
+                cursor=self.teradata_connection.cursor()
+                query_result=cursor.execute(single_query)
+
                 if return_as_pdf:
-                    query_result = pd.read_sql(query, self.teradata_connection)
-                else:
-                    cursor=self.teradata_connection.cursor()
-                    query_result=cursor.execute(single_query)
+                    query_result = pd.DataFrame(query_result)
 
                 results.append(query_result)
 

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation/services/result_service.py

@@ -81,7 +81,7 @@ class ResultService(TestingToolParams):
 
     @staticmethod
     def _compare_column_datatypes(
-            src_datatype: str, 
+            src_datatype: str,
             trgt_datatype: str
             ):
         """
@@ -96,7 +96,10 @@ class ResultService(TestingToolParams):
         elif "DATATYPE_MAPPING" in TestingToolParams.migration_config["MAPPING"] and TestingToolParams.migration_config["MAPPING"]["DATATYPE_MAPPING"]:
             datatype_equal = False
             for datatype_mapping in TestingToolParams.migration_config["MAPPING"]["DATATYPE_MAPPING"]:
-                if src_datatype in datatype_mapping["src_datatypes"] and trgt_datatype in datatype_mapping["trgt_datatypes"]:
+                if (
+                    src_datatype.lower() in [datatype.lower() for datatype in datatype_mapping["src_datatypes"]]
+                    and trgt_datatype.lower() in [datatype.lower() for datatype in datatype_mapping["trgt_datatypes"]]
+                    ):
                     datatype_equal = True
         else:
             datatype_equal = False
@@ -106,8 +109,8 @@ class ResultService(TestingToolParams):
 
     @staticmethod
     def prepare_column_level_result(
-            column: str, 
-            exclude_columns: list, 
+            column: str,
+            exclude_columns: list,
             result_params: ResultParams
         ) -> dict:
         """
@@ -122,7 +125,7 @@ class ResultService(TestingToolParams):
         aggregation_result_trgt = None
         aggregation_equal = None
         aggregation_tolerated = None
-        aggregation_difference_trgt_minus_src = None                   
+        aggregation_difference_trgt_minus_src = None
         count_nulls_src = None
         count_nulls_trgt = None
         count_nulls_equal = None
@@ -146,11 +149,11 @@ class ResultService(TestingToolParams):
             in_excluded = True
         else:
             in_excluded = False
-        
+
         if in_src and in_trgt :
             in_sync=True
             if result_params.src_columns_aggregate != {}:
-                
+
                 if column in result_params.src_columns_aggregate:
                     aggregation_type_src = result_params.src_columns_aggregate[column][0]
                     aggregation_result_src = result_params.src_columns_aggregate[column][1]
@@ -158,48 +161,48 @@ class ResultService(TestingToolParams):
 
                 if column in result_params.trgt_columns_aggregate:
                     aggregation_type_trgt = result_params.trgt_columns_aggregate[column][0]
-                    aggregation_result_trgt = result_params.trgt_columns_aggregate[column][1]                    
+                    aggregation_result_trgt = result_params.trgt_columns_aggregate[column][1]
                     count_nulls_trgt = result_params.trgt_columns_aggregate[column][2]
 
                 if column in result_params.aggregation_differences_trgt_minus_src and result_params.aggregation_differences_trgt_minus_src[column] and not result_params.aggregation_differences_trgt_minus_src[column] == '0_0':
                     aggregation_equal = False
                     aggregation_difference_trgt_minus_src = result_params.aggregation_differences_trgt_minus_src[column]
-                    
+
                 elif aggregation_result_src is not None and aggregation_result_trgt is not None and aggregation_type_src and aggregation_type_trgt and aggregation_type_src == aggregation_type_trgt:
                     aggregation_equal = True
-                        
+
                     if column in result_params.aggregation_differences_trgt_minus_src and result_params.aggregation_differences_trgt_minus_src[column] == '0_0':
                         aggregation_difference_trgt_minus_src='0_0'
-                        
-                    else: 
+
+                    else:
                         aggregation_difference_trgt_minus_src='0'
-                                    
+
                 if aggregation_type_src and aggregation_type_trgt and aggregation_type_src == aggregation_type_trgt:
                     aggregation_type = aggregation_type_src
-            
+
             '''
             Comparison Based on Decimal Places
             Logic is defined in migration_config.json
             '''
             aggregation_tolerated = aggregation_equal
-        
+
             if 'DATATYPE_TOLERANCE' in TestingToolParams.migration_config['MAPPING'].keys():
                 if (
-                    src_datatype in TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERANCE'].keys() 
-                    and aggregation_type == 'SUM' 
+                    src_datatype in TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERANCE'].keys()
+                    and aggregation_type == 'SUM'
                     and abs(Decimal(aggregation_difference_trgt_minus_src)) <= Decimal(TestingToolParams.migration_config['MAPPING']['DATATYPE_TOLERANCE'][src_datatype])
                     ):
                     aggregation_tolerated = True
-            else : 
+            else :
                 aggregation_tolerated = None
-                    
+
             if count_nulls_src is not None and count_nulls_trgt is not None and count_nulls_src==count_nulls_trgt:
                 count_nulls_equal = True
                 count_nulls_difference_trgt_minus_src = '0'
             elif count_nulls_src is not None and count_nulls_trgt is not None:
                     count_nulls_equal = False
                     count_nulls_difference_trgt_minus_src = int(count_nulls_trgt)-int(count_nulls_src)
-                
+
             datatype_equal = ResultService._compare_column_datatypes(src_datatype, trgt_datatype)
 
         column_comparison_result = {
@@ -224,15 +227,15 @@ class ResultService(TestingToolParams):
         }
 
         return column_comparison_result
-    
+
     @staticmethod
     def prepare_object_level_result(
-            src_object: DatabaseObject, 
-            trgt_object: DatabaseObject, 
-            src_filter: str, 
-            trgt_filter: str, 
-            exclude_columns: list, 
-            result_params: ResultParams, 
+            src_object: DatabaseObject,
+            trgt_object: DatabaseObject,
+            src_filter: str,
+            trgt_filter: str,
+            exclude_columns: list,
+            result_params: ResultParams,
             column_level_comparison_result: dict
         ) -> dict:
         """
@@ -268,9 +271,9 @@ class ResultService(TestingToolParams):
             "ROW_COUNTS_EQUAL": result_params.row_counts_equal,
             "SRC_ROW_COUNT": result_params.src_row_count,
             "TRGT_ROW_COUNT": result_params.trgt_row_count,
-            "ALL_COUNT_NULLS_EQUAL": result_params.all_count_nulls_equal,   
-            "AGGREGATIONS_EQUAL": result_params.aggregations_equal,  
-            "AGGREGATIONS_EQUAL_TOLERATED": aggregations_equal_tolerated,    
+            "ALL_COUNT_NULLS_EQUAL": result_params.all_count_nulls_equal,
+            "AGGREGATIONS_EQUAL": result_params.aggregations_equal,
+            "AGGREGATIONS_EQUAL_TOLERATED": aggregations_equal_tolerated,
             "SRC_ERROR": result_params.src_error_dict,
             "TRGT_ERROR":  result_params.trgt_error_dict,
             "GROUP_BY_COLUMNS": result_params.object_group_by_columns,
@@ -280,13 +283,13 @@ class ResultService(TestingToolParams):
             "GROUP_BY_VALUES_WITH_MISMATCHES": result_params.group_by_values_with_mismatches,
             "COLUMNS_WITH_MISMATCH": result_params.columns_with_mismatch,
             "GROUP_BY_DIFF_DICT": result_params.group_by_diff_dict,
-            "SRC_GROUP_BY_ERROR": result_params.src_group_by_error, 
+            "SRC_GROUP_BY_ERROR": result_params.src_group_by_error,
             "TRGT_GROUP_BY_ERROR": result_params.trgt_group_by_error,
-            "SAMPLES_COMPARED": result_params.samples_compared, 
-            "SAMPLES_EQUAL": result_params.samples_equal, 
-            "SAMPLE_KEYS": result_params.trgt_key_filters, 
-            "SRC_SAMPLE": result_params.src_sample_dict, 
-            "TRGT_SAMPLE": result_params.trgt_sample_dict, 
+            "SAMPLES_COMPARED": result_params.samples_compared,
+            "SAMPLES_EQUAL": result_params.samples_equal,
+            "SAMPLE_KEYS": result_params.trgt_key_filters,
+            "SRC_SAMPLE": result_params.src_sample_dict,
+            "TRGT_SAMPLE": result_params.trgt_sample_dict,
             "SRC_SAMPLE_QUERY": result_params.src_sample_query,
             "TRGT_SAMPLE_QUERY": result_params.trgt_sample_query,
             "SRC_SAMPLE_ERROR_DICT": result_params.src_sample_error_dict,
@@ -307,8 +310,8 @@ class ResultService(TestingToolParams):
 
     @staticmethod
     def prepare_object_level_live_result(
-            object_level_comparison_result: dict, 
-            testing_tool_params: TestingToolParams, 
+            object_level_comparison_result: dict,
+            testing_tool_params: TestingToolParams,
         ) -> dict:
         """
         Get object level live result dictionary from the object level comparison result and from the testing tool parameters.
@@ -324,7 +327,7 @@ class ResultService(TestingToolParams):
                 "OBJECTS": object_level_comparison_result
                 }
 
-        return  live_object_level_comparison_result   
+        return  live_object_level_comparison_result
 
     def determine_highlevel_results(self):
         """
@@ -368,9 +371,9 @@ class ResultService(TestingToolParams):
 
         # TODO add ALL_OBJECTS_NOT_ALTERED_DURING_COMPARISON flag
         #if any(object_level_comparison_result['NOT_ALTERED_DURING_COMPARISON_SRC'] == False for object_level_comparison_result in self.results["OBJECTS"]) or any(object_level_comparison_result['NOT_ALTERED_DURING_COMPARISON_TRGT'] == False for object_level_comparison_result in self.results["OBJECTS"]):
-        #    self.results["ALL_OBJECTS_NOT_ALTERED_DURING_COMPARISON"] = False 
+        #    self.results["ALL_OBJECTS_NOT_ALTERED_DURING_COMPARISON"] = False
         #else:
-        #    self.results["ALL_OBJECTS_NOT_ALTERED_DURING_COMPARISON"] = True   
+        #    self.results["ALL_OBJECTS_NOT_ALTERED_DURING_COMPARISON"] = True
 
         logger.info("\n****************************************************")
         logger.info(f"++++++++++++++++ Highlevel results  ++++++++++++++++")
@@ -383,8 +386,8 @@ class ResultService(TestingToolParams):
         logger.info(f"ALL_CHECKSUMS_EQUAL: {self.results['ALL_CHECKSUMS_EQUAL']}")
         logger.info(f"ALL_SAMPLES_EQUAL: {self.results['ALL_SAMPLES_EQUAL']}")
         logger.info(f"ALL_OBJECTS_EQUAL: {self.results['ALL_OBJECTS_EQUAL']}")
-        logger.info("****************************************************\n") 
-            
+        logger.info("****************************************************\n")
+
 
     def load_results_to_result_database(self):
         """
@@ -400,12 +403,12 @@ class ResultService(TestingToolParams):
         with database_service_result as db_service_result:
 
             load_results_function = self.load_results_function_mapping[result_system_selection_type.upper()]
-            
+
             if load_results_function:
                 load_results_function(db_service_result, self.results)
             else:
                 raise ValueError(f"Result system selection of type '{result_system_selection_type}' not supported!")
-    
+
     def load_results_to_snowflake(self, db_service_result, results: dict):
         """
         Load results to Snowflake.
@@ -474,9 +477,9 @@ class ResultService(TestingToolParams):
         blob_file_name = f"comparison_results_{start_time_utc}_{self.pipeline_name}_{self.pipeline_id}_{self.run_guid}.json"
         blob_name = f"{blob_file_prefix}/{blob_file_name}"
 
-        try: 
+        try:
             blob_service_client = BlobServiceClient.from_connection_string(conn_str=self.azure_storage_connection_string)
-        except Exception as error:    
+        except Exception as error:
             logger.info(f"FAILED to connect to Azure Blob Storage with error '{str(error)}'")
             raise error
 
@@ -510,13 +513,13 @@ class ResultService(TestingToolParams):
         bucket_file_info = f"comparison_results_{start_time_utc}_{self.pipeline_name}_{self.pipeline_id}_{self.run_guid}.json"
         bucket_file_name = f"{bucket_file_prefix}_-_{bucket_file_info}"
 
-        try: 
+        try:
             s3_service_client = boto3.client(
                 's3',
                 aws_access_key_id=self.aws_bucket_access_key,
                 aws_secret_access_key=self.aws_bucket_secret_key
             )
-        except Exception as error:    
+        except Exception as error:
             logger.info(f"FAILED to connect to AWS S3 bucket with error '{str(error)}'")
             raise error
 
@@ -533,7 +536,7 @@ class ResultService(TestingToolParams):
 
     def write_results_to_git(self):
         """
-        Write comparison results to GIT repository. 
+        Write comparison results to GIT repository.
         In case of a remote pipeline run: Pull latest changes from GIT befor writing to the local repository, and push to the remote repository at the end.
         """
         logger.info(f"++++++++++++++++ WRITE comparison results to GIT repository")

{icsdatavalidation-1.0.425 → icsdatavalidation-1.0.428}/icsDataValidation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: icsDataValidation
-Version: 1.0.425
+Version: 1.0.428
 Summary: Add your description here
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT