icsDataValidation 1.0.428__py3-none-any.whl → 1.0.438__py3-none-any.whl

icsDataValidation/services/database_services/sqlserver_service.py

@@ -1,9 +1,7 @@
-import pyodbc
-import pandas.io.sql
 import logging
-import pandas as pd
 
-from pathlib import PurePath
+import pandas as pd
+import pyodbc
 
 from icsDataValidation.core.database_objects import DatabaseObject
 from icsDataValidation.utils.logger_util import configure_dev_ops_logger
@@ -38,7 +36,11 @@ class SQLServerService:
 
     def __del__(self):
         if self.sqlserver_connection is not None:
-            self.sqlserver_connection.close()
+            try:
+                self.sqlserver_connection.close()
+            except pyodbc.Error:
+                pass  # Connection might already be closed
+            self.sqlserver_connection = None
 
     def _connect_to_sqlserver(self):
         sqlserver_connection_string = (
@@ -47,8 +49,12 @@ class SQLServerService:
             f"PORT={self.connection_params['Port']};"
             f"DATABASE={self.connection_params['Database']};"
             f"UID={self.connection_params['User']};"
-            f"PWD={self.connection_params['Password']}"
+            f"PWD={self.connection_params['Password']};"
         )
+        if self.connection_params["Encrypt"] is True:
+            sqlserver_connection_string += "Encrypt=Yes;"
+        if self.connection_params["TrustServerCertificate"] is True:
+            sqlserver_connection_string += "TrustServerCertificate=Yes;"
         self.sqlserver_connection = pyodbc.connect(sqlserver_connection_string)
         return self.sqlserver_connection
 
@@ -81,20 +87,25 @@ class SQLServerService:
             str: in clause as string
         """
         values = list(key_filters.values())
-        in_clause_values = "'"
+        in_clause_values = ""
+
+        if len(values) == 0:
+            return in_clause_values
+
         for j in range(len(values[0])):
-            for value in values:
-                in_clause_values += str(value[j]) + "','"
-            in_clause_values = in_clause_values[:-2] + ",'"
-        in_clause_values = in_clause_values[:-3] + "'"
+            sample_j = list(map(lambda arr:  arr[j] , values))
+            in_clause_values += "'" + "|".join(str(x) for x in sample_j) + "|',"
+        in_clause_values = in_clause_values[:-1]
 
-        in_clause_cols = " AND (("
+
+        in_clause_cols = " AND (CONCAT("
         for key in key_filters.keys():
             if key in numeric_columns:
-                in_clause_cols += f"""cast(ROUND({key.replace("'", "")}, {numeric_scale}) as numeric(38, {numeric_scale}))""" + ","
+                in_clause_cols += f"""cast(ROUND([{key.replace("'", "")}], {numeric_scale}) as numeric(38, {numeric_scale}))""" + ", '|' ,"
             else:
-                in_clause_cols += key.replace("'", "") + ","
-        in_clause_cols = in_clause_cols[:-1] + ")"
+                in_clause_cols += f"""[{key.replace("'", "")}], '|' ,"""
+
+        in_clause_cols = in_clause_cols[:-2] + ")"
         in_clause = in_clause_cols + " in (" + in_clause_values + "))"
         return in_clause
 
@@ -123,14 +134,14 @@ class SQLServerService:
                 if column_datatype.lower() in self.sqlserver_datatype_mapping["numeric"]:
                     if numeric_scale:
                         column_intersecions_new.append(
-                            f"CAST(ROUND({column}, {numeric_scale}) as decimal(38,{numeric_scale})) as {column}"
+                            f"CAST(ROUND([{column}], {numeric_scale}) as decimal(38,{numeric_scale})) as [{column}]"
                         )
                     else:
-                        column_intersecions_new.append(f"{column} as {column}")
+                        column_intersecions_new.append(f"[{column}] as [{column}]")
                     used_columns.append(column)
                     numeric_columns.append(column)
                 elif column_datatype.lower() in self.sqlserver_datatype_mapping["string"]:
-                    column_intersecions_new.append(f"{column} AS {column}")
+                    column_intersecions_new.append(f"[{column}] AS [{column}]")
                     used_columns.append(column)
                 else:
                     column_intersecions_new.append(column)
@@ -140,6 +151,97 @@ class SQLServerService:
         column_clause = str(column_intersections)[1:-1].replace("'", "")
         return column_clause, numeric_columns, used_columns
 
+    def _get_checksum_statement(self,
+        object: DatabaseObject,
+        column_intersections: list,
+        where_clause: str = "",
+        exclude_columns: list = [],
+        numeric_scale: int = None,
+        bool_cast_before_sum: bool = False) -> str:
+        """
+        Creates checksum sql statement for given object in compliance with given conditions
+
+        object (DatabaseObject): table or view
+        column_intersections (list): columns that are used for checksums
+        where_clause (str, optional): Optional filter criteria given as sql-usable string
+        exclude_columns (list, optional): columns to exlude from calculation
+        numeric_scale (int, optional): number of decimal places for aggregations
+        bool_cast_before_sum (bool, optional): whether to cast before sum
+
+        Returns:
+            str: checksum sql statement
+        """
+
+        column_intersections = [f"{x.upper()}" for x in column_intersections if x not in exclude_columns]
+        dict_colummns_datatype = self.get_data_types_from_object(object, column_intersections)
+        aggregates = ""
+
+        for column in column_intersections:
+            column_datatype = next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
+
+            if column_datatype.lower() in self.sqlserver_datatype_mapping["numeric"]:
+                if not bool_cast_before_sum:
+                    if numeric_scale:
+                        aggregates += (
+                            f", CAST(ROUND(SUM([{column}]), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS [SUM_{column}]"
+                        )
+                    else:
+                        aggregates += f", CAST(SUM([{column}]) AS DECIMAL(38)) AS [SUM_{column}]"
+                else:
+                    if numeric_scale:
+                        aggregates += (
+                            f", ROUND(SUM(CAST([{column}] AS DECIMAL(38, {numeric_scale}))), {numeric_scale}) AS [SUM_{column}]"
+                        )
+                    else:
+                        aggregates += f", SUM(CAST([{column}] AS DECIMAL(38))) AS [SUM_{column}]"
+            elif (
+                column_datatype.lower() in self.sqlserver_datatype_mapping["string"]
+                or column_datatype.lower() in self.sqlserver_datatype_mapping["date_and_time"]
+            ):
+                aggregates += f", COUNT(DISTINCT LOWER([{column}])) AS [COUNTDISTINCT_{column}]"
+
+            elif column_datatype.lower() in self.sqlserver_datatype_mapping["binary"]:
+                aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,[{column}]))) AS [COUNTDISTINCT_{column}]"
+
+            elif column_datatype.lower() in self.sqlserver_datatype_mapping["boolean"]:
+                aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN [{column}] = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN [{column}] = 0 THEN 1 ELSE NULL END))) AS [AGGREGATEBOOLEAN_{column}]"
+
+            #else: Additional Data Types: image , sql_variant, uniqueidentifier, xml, cursor, table, column_datatype.lower() == 'bit' or
+        query_checksums = (
+            f"SELECT {aggregates[1:]} FROM {object.schema}.{object.name} {where_clause};"
+        )
+
+        return query_checksums
+
+    def _get_countnulls_statement(self,
+        object: DatabaseObject,
+        column_intersections: list,
+        where_clause: str = "",
+        exclude_columns: list = []):
+        """
+        Creates countnulls sql statement for given object in compliance with given conditions
+
+        object (DatabaseObject): table or view
+        column_intersections (list): columns that are used for checksums
+        where_clause (str, optional): Optional filter criteria given as sql-usable string
+        exclude_columns (list, optional): columns to exlude from calculation
+
+        Returns:
+            str: countnulls sql statement
+        """
+        column_intersections = [f"{x.upper()}" for x in column_intersections if x not in exclude_columns]
+        count_nulls = ""
+
+        for column in column_intersections:
+            count_nulls += f", SUM(CASE WHEN [{column}] IS NULL THEN 1 ELSE 0 END) AS [COUNTNULLS_{column}]"
+
+        query_countnulls = (
+            f"SELECT {count_nulls[1:]} FROM {object.schema}.{object.name} {where_clause};"
+        )
+
+        return query_countnulls
+
+
     def get_database_objects(
         self, database: str, schema: str = None, object_type_restriction: str = "include_all"
     ) -> dict:
@@ -376,7 +478,7 @@ class SQLServerService:
                             UNION
                                 SELECT
                                     '{column}' AS COLUMN_NAME,
-                                      COUNT(DISTINCT {column}) AS COUNT_DISTINCT
+                                      COUNT(DISTINCT [{column}]) AS COUNT_DISTINCT
                                 FROM {object.schema}.{object.name}
                                 {where_clause}
                         """
@@ -429,6 +531,7 @@ class SQLServerService:
 
         return size
 
+
     def create_checksums(
         self,
         object: DatabaseObject,
@@ -454,46 +557,21 @@ class SQLServerService:
         if self.sqlserver_connection is None:
             self._connect_to_sqlserver()
 
-        column_intersections = [f"{x.upper()}" for x in column_intersections if x not in exclude_columns]
-
-        dict_colummns_datatype = self.get_data_types_from_object(object, column_intersections)
-
-        aggregates = ""
-        count_nulls = ""
-
-        for column in column_intersections:
-            column_datatype = next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
-
-            count_nulls += f", SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END) AS COUNTNULLS_{column}"
-
-            if column_datatype.lower() in self.sqlserver_datatype_mapping["numeric"]:
-                if numeric_scale:
-                    aggregates += (
-                        f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38, {numeric_scale})) AS SUM_{column}"
-                    )
-                else:
-                    aggregates += f", CAST(SUM({column}) AS DECIMAL(38)) AS SUM_{column}"
-
-            elif (
-                column_datatype.lower() in self.sqlserver_datatype_mapping["string"]
-                or column_datatype.lower() in self.sqlserver_datatype_mapping["date_and_time"]
-            ):
-                aggregates += f", COUNT(DISTINCT LOWER({column})) AS COUNTDISTINCT_{column}"
-
-            elif column_datatype.lower() in self.sqlserver_datatype_mapping["binary"]:
-                aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,{column}))) AS COUNTDISTINCT_{column}"
-
-            elif column_datatype.lower() in self.sqlserver_datatype_mapping["boolean"]:
-                aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END))) AS AGGREGATEBOOLEAN_{column}"
-
-            #else: Additional Data Types: image , sql_variant, uniqueidentifier, xml, cursor, table, column_datatype.lower() == 'bit' or
-
-        query_checksums = (
-            f"SELECT {aggregates[1:]} FROM {object.schema}.{object.name} {where_clause};"
+        ## get checksum query
+        query_checksums = self._get_checksum_statement(
+            object=object,
+            column_intersections=column_intersections,
+            where_clause=where_clause,
+            exclude_columns=exclude_columns,
+            numeric_scale=numeric_scale,
         )
 
-        query_countnulls = (
-            f"SELECT {count_nulls[1:]} FROM {object.schema}.{object.name} {where_clause};"
+        ## get countnulls query
+        query_countnulls = self._get_countnulls_statement(
+            object=object,
+            column_intersections=column_intersections,
+            where_clause=where_clause,
+            exclude_columns=exclude_columns
         )
 
         error_list = []
@@ -502,28 +580,50 @@ class SQLServerService:
 
         try:
             checksums_results = self.execute_queries([query_checksums, query_countnulls])
-
             aggregation_results = checksums_results[0][0]
-
             countnulls_results = checksums_results[1][0]
-
-            for i in range(0, len(aggregation_results)):
-                if list(aggregation_results.values())[i] is None:
-                    agg_result = 0
-                else:
-                    agg_result = list(aggregation_results.values())[i]
-
-                if list(countnulls_results.values())[i] is None:
-                    cnt_result = 0
-                else:
-                    cnt_result = list(countnulls_results.values())[i]
-
-                test_list.append(
-                    [[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i], agg_result, cnt_result]
+        except Exception as err:
+            err_msg = ["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]]
+
+            if 'Arithmetic overflow' in err_msg[2]:
+                # re-calculate queries with bool_cast_before_sum=True in case of error
+                query_checksums = self.create_checksum_statement(
+                    object=object,
+                    column_intersections=column_intersections,
+                    where_clause=where_clause,
+                    exclude_columns=exclude_columns,
+                    numeric_scale=numeric_scale,
+                    bool_cast_before_sum=True
                 )
+                try:
+                    # if overflow then try again with cast before sum for booleans
+                    checksums_results = self.execute_queries([query_checksums, query_countnulls])
+                    aggregation_results = checksums_results[0][0]
+                    countnulls_results = checksums_results[1][0]
+                except Exception as err:
+                    # handle error if it still occurs
+                    err_msg = ["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]]
+                    error_list.append(err_msg)
+            else:
+                # handle error if it is not an overflow
+                error_list.append(err_msg)
+                checksums_results = None
+
+        # if error occured before this will be skipped as aggregation_results would be empty
+        for i in range(0, len(aggregation_results)):
+            if list(aggregation_results.values())[i] is None:
+                agg_result = 0
+            else:
+                agg_result = list(aggregation_results.values())[i]
 
-        except Exception as err:
-            error_list.append(["ERROR", str(err).split("|||")[0], str(err).split("|||")[1]])
+            if list(countnulls_results.values())[i] is None:
+                cnt_result = 0
+            else:
+                cnt_result = list(countnulls_results.values())[i]
+
+            test_list.append(
+                [[item.split("_", 1)[0] for item in list(aggregation_results.keys())][i], agg_result, cnt_result]
+            )
 
         checksums = dict(zip([item.split("_", 1)[1] for item in aggregation_results.keys()], test_list))
         checksums["TESTATM_ERRORS"] = error_list
@@ -583,7 +683,7 @@ class SQLServerService:
         try:
             for column in group_by_columns:
                 if column in column_intersections and column not in exclude_columns:
-                    group_by_query_columns_string += f"{column} ,"
+                    group_by_query_columns_string += f"[{column}] ,"
                     grouping_columns_final.append(column)
 
             group_by_query_columns_string = group_by_query_columns_string[:-1]
@@ -598,23 +698,24 @@ class SQLServerService:
 
                 if column_datatype.lower() in self.sqlserver_datatype_mapping["numeric"]:
                     if numeric_scale:
-                        aggregates_min += f", CAST(ROUND(MIN({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MIN_{column}, CAST(ROUND(MAX({column}),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS MAX_{column}"
-                        aggregates += f", CAST(ROUND(SUM({column}), {numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS SUM_{column}"
+                        aggregates_min += f""", CAST(ROUND(MIN([{column}]),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS [MIN_{column}[]
+                                            , CAST(ROUND(MAX([{column}]),{numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS [MAX_{column}]"""
+                        aggregates += f", CAST(ROUND(SUM([{column}]), {numeric_scale}) AS DECIMAL(38,{numeric_scale})) AS [SUM_{column}]"
                     else:
-                        aggregates_min += f", MIN({column}) AS MIN_{column}, MAX({column}) AS MAX_{column}"
-                        aggregates += f", SUM({column}) AS SUM_{column}"
-
+                        aggregates_min += f", MIN([{column}]) AS [MIN_{column}], MAX([{column}]) AS [MAX_{column}]"
+                        aggregates += f", SUM([{column}]) AS [SUM_{column}]"
                 elif not only_numeric and (
                     column_datatype.lower() in self.sqlserver_datatype_mapping["string"]
                     or column_datatype.lower() in self.sqlserver_datatype_mapping["date_and_time"]
                 ):
-                    aggregates += f", COUNT(DISTINCT LOWER({column})) AS COUNTDISTINCT_{column}"
+                    aggregates += f", COUNT(DISTINCT LOWER([{column}])) AS [COUNTDISTINCT_{column}]"
 
                 elif not only_numeric and column_datatype.lower() in self.sqlserver_datatype_mapping["binary"]:
-                    aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,{column}))) AS COUNTDISTINCT_{column}"
+                    aggregates += f", COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,[{column}]))) AS [COUNTDISTINCT_{column}]"
 
                 elif not only_numeric and column_datatype.lower() in self.sqlserver_datatype_mapping["boolean"]:
-                    aggregates += f", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN {column} = 1 THEN 1 ELSE NULL END)) , '_'),  CONVERT(VARCHAR, COUNT(CASE WHEN {column} = 0 THEN 1 ELSE NULL END))) AS AGGREGATEBOOLEAN_{column}"
+                    aggregates += f""", CONCAT(CONCAT(CONVERT(VARCHAR,COUNT(CASE WHEN [{column}] = 1 THEN 1 ELSE NULL END)) , '_')
+                    ,  CONVERT(VARCHAR, COUNT(CASE WHEN [{column}] = 0 THEN 1 ELSE NULL END))) AS [AGGREGATEBOOLEAN_{column}]"""
 
                 # else: Additional Data Types: VARIANT OBJECT ARRAY GEOGRAPHY
 
@@ -630,7 +731,13 @@ class SQLServerService:
             elif group_by_aggregation_type == "various_and_min_max":
                 group_by_query_aggregation_string = f"{aggregates_min[1:]}{aggregates}"
 
-            query_group_by_aggregation = f"SELECT {group_by_query_columns_string}, COUNT(*) AS COUNT_OF_GROUP_BY_VALUE, {group_by_query_aggregation_string} FROM {object.database}.{object.schema}.{object.name} {where_clause} GROUP BY {group_by_query_columns_string} ORDER BY {group_by_query_columns_string};"
+            query_group_by_aggregation = f""" \
+                SELECT {group_by_query_columns_string} \
+                , COUNT(*) AS COUNT_OF_GROUP_BY_VALUE\
+                {', '+ group_by_query_aggregation_string if group_by_query_aggregation_string != '' else ''}\
+                FROM {object.schema}.{object.name} {where_clause}\
+                GROUP BY {group_by_query_columns_string}\
+                ORDER BY {group_by_query_columns_string};"""
 
             group_by_aggregation_pdf = self.execute_queries(query_group_by_aggregation, True)
         except Exception as err:
@@ -680,7 +787,8 @@ class SQLServerService:
         if self.sqlserver_connection is None:
             self._connect_to_sqlserver()
 
-        intersection_columns_trgt_src_ = ", ".join(list(set(intersection_columns_trgt_src) - set(exclude_columns)))
+        col_list_enclosed = [f"[{col}]" for col in list(set(intersection_columns_trgt_src) - set(exclude_columns))]
+        intersection_columns_trgt_src_ = ", ".join(col_list_enclosed)
 
         df_query = f"SELECT {intersection_columns_trgt_src_} FROM {object.schema}.{object.name} {where_clause};"
 
@@ -727,7 +835,7 @@ class SQLServerService:
             dict_colummns_datatype = self.get_data_types_from_object(object, column_intersections)
 
         if key_intersection != [] and is_dedicated:
-            keys = str(key_intersection)[1:-1].replace("'", "")
+            keys = str([f"""[{key}]""" for key in key_intersection])[1:-1].replace("'", "")
             column_clause, numeric_columns, used_columns = self._get_column_clause(
                 dedicated_intersection, dict_colummns_datatype, numeric_scale, key_columns,
                 enclose_column_by_double_quotes
@@ -747,7 +855,7 @@ class SQLServerService:
                                 ORDER BY {keys};
                             """
         elif key_intersection != [] and not is_dedicated:
-            keys = str(key_intersection)[1:-1].replace("'", "")
+            keys = str([f"""[{key}]""" for key in key_intersection])[1:-1].replace("'", "")
             column_clause, numeric_columns, used_columns = self._get_column_clause(
                 column_intersections, dict_colummns_datatype, numeric_scale, key_columns,
                 enclose_column_by_double_quotes

{icsdatavalidation-1.0.428.dist-info → icsdatavalidation-1.0.438.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: icsDataValidation
-Version: 1.0.428
+Version: 1.0.438
 Summary: Add your description here
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT

{icsdatavalidation-1.0.428.dist-info → icsdatavalidation-1.0.438.dist-info}/RECORD

@@ -1,18 +1,18 @@
 icsDataValidation/configuration.py,sha256=HOFjmC8_e2nvoItndMtJQQA1MR5aCgZGeF1AwY_FvjE,477
-icsDataValidation/main.py,sha256=1CtzUa0-LALTH9i5eQ6H6PW0UON3TPHZ5ey9qDsljKQ,11502
+icsDataValidation/main.py,sha256=EztJRS1UMIJ3vikjzOnDJ9ef3zgrmCSGXoyuAhJjudM,11501
 icsDataValidation/connection_setups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/connection_setups/azure_connection_setup.py,sha256=qxPvD-VZhdJqrdj06IVIk2Ud287YlLhE22Q5_oYKetM,790
 icsDataValidation/connection_setups/databricks_connection_setup.py,sha256=dNEBum-8R-TUW2SCEk3CaNtCr_gLFvn456KBlENpgJU,1220
 icsDataValidation/connection_setups/exasol_connection_setup.py,sha256=RfCUsL6G-NaOW-qNK-3SfHcljbRaKD6fDIHXkNQhClk,590
 icsDataValidation/connection_setups/oracle_connection_setup.py,sha256=D-4ucC1ChE4HYm93ECIEg_yBOrn1NkknxFBgFRGFmWs,978
 icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=IgEhni4Q0oYGh2QzptpyfEUvUt3cVO28jNSGg11cxyI,1778
-icsDataValidation/connection_setups/sqlserver_connection_setup.py,sha256=ayRao5BbhkEJTteaeZiryz_GLC_6F_02XalvJDHM_4k,802
+icsDataValidation/connection_setups/sqlserver_connection_setup.py,sha256=Lg4jh0NxujcpGWzO3BKdWP5cS742smcqVtvGjPOBq1A,910
 icsDataValidation/connection_setups/teradata_connection_setup.py,sha256=fIpuxz-FTqFK2vSMSuokqU9sdJkaJ4UP5piY_zIbj5k,624
 icsDataValidation/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/core/database_objects.py,sha256=2oaDaVQajSYI_HJjJy1pmc6FsoK_wMfwgu6ZgEcFvow,523
 icsDataValidation/core/object_comparison.py,sha256=xJvgHdoRaMzFMQishpzEszO7bW31Ll9BUCsyzqwrRVs,15045
 icsDataValidation/input_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-icsDataValidation/input_parameters/testing_tool_params.py,sha256=trVZmxd2hsJRBWgdv0YePdaA9T20QbL3bOCVUOwIH18,6907
+icsDataValidation/input_parameters/testing_tool_params.py,sha256=9MPEF4BrT-twmt4gLE2VRrhD9o59JbXOhwfeqx5qlVA,6721
 icsDataValidation/output_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/output_parameters/result_params.py,sha256=HLS7DUX8NWWw3j5de8qOQ4T4auWbyMuwmuafzaBOjnU,2861
 icsDataValidation/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,8 +27,8 @@ icsDataValidation/services/database_services/databricks_hive_metastore_service.p
 icsDataValidation/services/database_services/databricks_unity_catalog_service.py,sha256=8iV75kvtQsGPdC35m89jO5s0ZQDekPdRVPYGbdCAPVI,70835
 icsDataValidation/services/database_services/exasol_service.py,sha256=LdjU8mM77zTmNmhJPQrgQO-HwAZv0C0seYMDjuWU9BQ,11153
 icsDataValidation/services/database_services/oracle_service.py,sha256=Ejxi0HBRF_c0xWY4wEsw8L8Rb5FMRf9cjQbhz8kerIA,31805
-icsDataValidation/services/database_services/snowflake_service.py,sha256=KCKV-rYR8w2ml1ygtxFxph8ScIB_37GyAttszBwS-wY,63321
-icsDataValidation/services/database_services/sqlserver_service.py,sha256=FZAPiBm_l7-cuUh1Ef88xwV0Gmur_Zo9oar9nukaDuI,38401
+icsDataValidation/services/database_services/snowflake_service.py,sha256=hygxlqvLw-6PLJFoATsp5zHup4vcuGAOvEAzgkLBkXw,68657
+icsDataValidation/services/database_services/sqlserver_service.py,sha256=6FD6vp8K3bMkfQSaCLuPEtR3KMc0IsvVAybRDwsg3Po,43303
 icsDataValidation/services/database_services/teradata_service.py,sha256=2x7onntG5E1qqw65HXUmFwcrYmT5I8HSS3eWXIhTfiw,40252
 icsDataValidation/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 icsDataValidation/utils/file_util.py,sha256=ZTMB1sTnIIdffg9tEJRCFQQ5SG8Fksc5ie1PM4gHXG4,3432
@@ -36,7 +36,21 @@ icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI
 icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
 icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
 icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
-icsdatavalidation-1.0.428.dist-info/METADATA,sha256=XxIA3hQlU_s84FdMBTGXk-ITuaVtva0Rp9g5x_SpZf0,661
-icsdatavalidation-1.0.428.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-icsdatavalidation-1.0.428.dist-info/top_level.txt,sha256=BqWUGJb4J7ZybpDMeuGHxEHGHwXXJEIURd9pBybHzTM,18
-icsdatavalidation-1.0.428.dist-info/RECORD,,
+tests/snowflake_service/test_create_checksums.py,sha256=ifXxuNk7RHg5haTznBDsnJM9lrxXyBssVICn_oWkgj8,5397
+tests/snowflake_service/test_create_pandas_df_from_group_by.py,sha256=7ZF-RbloV6kZTirWdBRNkpfhNeER7vOQE83BGILbXHk,17013
+tests/snowflake_service/test_create_pandas_df_from_sample.py,sha256=z3-G_cctDttL7SR8eUep421xGBo54t3duWjeHpzt7LI,16794
+tests/snowflake_service/test_get_checksum_statement.py,sha256=U9IZqjr5YzwGKcVU6Q4h1l1G-CbwX0STQeFdEvOqV0c,8313
+tests/snowflake_service/test_get_column_clause.py,sha256=5ZIMvElXWfnjBUrYaEMDFzIpEcMO4oXJEoO82QObTz0,11019
+tests/snowflake_service/test_get_countnulls_statement.py,sha256=l1hah4oVKp983IdWUfO08ojgmgy6hmfVopn6xW1cXu4,4254
+tests/snowflake_service/test_get_in_clause.py,sha256=z0IkJhh1XSQ6rIB0VL1RBc616i1BvXjzJqZ6_ijfToU,2442
+tests/sqlserver_service/test_create_checksums.py,sha256=aknl9JzfDCHomIYlNcnp-zNYoinJcZ-rgTGUJoVC4Zs,5557
+tests/sqlserver_service/test_create_pandas_df_from_group_by.py,sha256=hFZCyHep3jiVdrFGQF3eWugXK2tCv75E-WkGoU_-JsY,15071
+tests/sqlserver_service/test_create_pandas_df_from_sample.py,sha256=d4XG4O6JcXIW1JlA2E9hJw_FXlH6zRkQP_K6lUWUOl0,10818
+tests/sqlserver_service/test_get_checksum_statement.py,sha256=IZq1lwS5IEoBqNcioLq1w8mjRuCTq6Uwc27xD9OD8PA,5422
+tests/sqlserver_service/test_get_column_clause.py,sha256=JKgglx_KJTbPeVfmSMdbl2wiSwu3_R6fZs7a36LlO0Q,6232
+tests/sqlserver_service/test_get_countnulls_statement.py,sha256=ZwowVsdAuYAMeGgTk4puJMDDXlETJngTygeesqsKv7w,4003
+tests/sqlserver_service/test_get_in_clause.py,sha256=Ee4kAZdbxMQ_evlJscV8DwKtjbuEeRYUt2PaxZHLoXA,3160
+icsdatavalidation-1.0.438.dist-info/METADATA,sha256=UthzSwJ3xyrArmNECa3jdZ75wdAA9qh2hERHrWFjEqY,661
+icsdatavalidation-1.0.438.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+icsdatavalidation-1.0.438.dist-info/top_level.txt,sha256=y0PjCpmJ_Vhc0QB0SgXxxcRSR7__mQV5rmFyfQc60nA,24
+icsdatavalidation-1.0.438.dist-info/RECORD,,

{icsdatavalidation-1.0.428.dist-info → icsdatavalidation-1.0.438.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{icsdatavalidation-1.0.428.dist-info → icsdatavalidation-1.0.438.dist-info}/top_level.txt

@@ -1 +1,2 @@
 icsDataValidation
+tests

tests/snowflake_service/test_create_checksums.py

@@ -0,0 +1,146 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+@pytest.fixture
+def snowflake_service():
+    """Create a SnowflakeService instance with mocked connection."""
+    mock_params = MagicMock()
+    service = SnowflakeService(mock_params)
+    service.snowflake_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    return DatabaseObject(
+        object_identifier="TEST_DB.TEST_SCHEMA.TEST_TABLE",
+        object_type=DatabaseObjectType.TABLE
+    )
+
+
+class TestCreateChecksumsParametrized:
+    """Parametrized tests for SnowflakeService.create_checksums."""
+
+    @pytest.mark.parametrize(
+        "column_intersections,where_clause,numeric_scale,execute_behavior," \
+        "expected_columns,expected_errors,expect_retry,expected_execute_calls",
+        [
+            (  # success path
+                ['amount', 'name'],
+                'WHERE amount > 0',
+                2,
+                {
+                    "return_value": [
+                        [{'SUM_AMOUNT': 10, 'COUNTDISTINCT_NAME': 3}],
+                        [{'COUNTNULLS_AMOUNT': 1, 'COUNTNULLS_NAME': 0}]
+                    ]
+                },
+                {
+                    'AMOUNT': ['SUM', 10, 1],
+                    'NAME': ['COUNTDISTINCT', 3, 0]
+                },
+                [],
+                False,
+                1
+            ),
+            (  # arithmetic overflow triggers retry
+                ['amount'],
+                '',
+                None,
+                {
+                    "side_effect": [
+                        Exception('checksum_sql|||Arithmetic overflow error converting numeric to data type numeric'),
+                        [[{'SUM_AMOUNT': 5}], [{'COUNTNULLS_AMOUNT': 0}]]
+                    ]
+                },
+                {
+                    'AMOUNT': ['SUM', 5, 0]
+                },
+                [],
+                True,
+                2
+            ),
+            (  # non-overflow error surfaces in TESTATM_ERRORS
+                ['amount'],
+                '',
+                None,
+                {
+                    "side_effect": Exception('checksum_sql|||Some other error')
+                },
+                {},
+                [['ERROR', 'checksum_sql', 'Some other error']],
+                False,
+                1
+            ),
+        ],
+    )
+    def test_create_checksums(
+        self,
+        snowflake_service,
+        mock_database_object,
+        column_intersections,
+        where_clause,
+        numeric_scale,
+        execute_behavior,
+        expected_columns,
+        expected_errors,
+        expect_retry,
+        expected_execute_calls
+    ):
+        """Test create_checksums behavior across success, retry, and error scenarios."""
+        snowflake_service.create_checksum_statement = MagicMock(return_value='checksum_retry_sql')
+
+        with patch.object(snowflake_service, '_get_checksum_statement', return_value='checksum_sql') as mock_checksum_stmt, \
+             patch.object(snowflake_service, '_get_countnulls_statement', return_value='countnulls_sql') as mock_countnulls_stmt, \
+             patch.object(snowflake_service, 'execute_queries') as mock_execute:
+
+            if 'side_effect' in execute_behavior:
+                mock_execute.side_effect = execute_behavior['side_effect']
+            else:
+                mock_execute.return_value = execute_behavior['return_value']
+
+            result = snowflake_service.create_checksums(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[],
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=False
+            )
+
+            mock_checksum_stmt.assert_called_once_with(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[],
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=False
+            )
+            mock_countnulls_stmt.assert_called_once_with(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[],
+                enclose_column_by_double_quotes=False
+            )
+            assert mock_execute.call_count == expected_execute_calls
+
+            if expect_retry:
+                snowflake_service.create_checksum_statement.assert_called_once()
+                retry_kwargs = snowflake_service.create_checksum_statement.call_args.kwargs
+                assert retry_kwargs['bool_cast_before_sum'] is True
+            else:
+                snowflake_service.create_checksum_statement.assert_not_called()
+
+            for column, expected in expected_columns.items():
+                assert result[column] == expected
+
+            expected_keys = set(expected_columns.keys()) | {'TESTATM_ERRORS'}
+            assert set(result.keys()) == expected_keys
+            assert result['TESTATM_ERRORS'] == expected_errors