icsDataValidation 1.0.428__py3-none-any.whl → 1.0.438__py3-none-any.whl

tests/snowflake_service/test_create_pandas_df_from_sample.py

@@ -0,0 +1,444 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+@pytest.fixture
+def snowflake_service():
+    """Create a SnowflakeService instance with mocked connection."""
+    mock_params = MagicMock()
+    service = SnowflakeService(mock_params)
+    service.snowflake_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    obj = DatabaseObject(
+        object_identifier="TEST_DB.TEST_SCHEMA.TEST_TABLE",
+        object_type=DatabaseObjectType.TABLE
+    )
+    return obj
+
+
+class TestCreatePandasDfFromSampleParametrized:
+    """Parametrized tests for create_pandas_df_from_sample method."""
+
+    @pytest.mark.parametrize(
+        "column_intersections,key_columns,exclude_columns,dedicated_columns," \
+        "where_clause,key_filters,sample_count,numeric_scale,enclose_quotes," \
+        "mock_datatypes,mock_column_clause,mock_in_clause,expected_contains,expected_not_in",
+        [
+            ( # simple case with key columns, no double quotes
+                ['id', 'name', 'amount'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ("id as id, name AS name, amount as amount", ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                ["SELECT id as id, name AS name, amount as amount", "FROM TEST_DB.TEST_SCHEMA.TEST_TABLE", "SAMPLE (10 ROWS)", "WHERE 1=1", "ORDER BY id;"],
+                []
+            ),
+            ( # simple case with key columns, with double quotes
+                ['id', 'name', 'amount'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ('"id" as "id", "name" AS "name", "amount" as "amount"', ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                ['SELECT "id" as "id", "name" AS "name", "amount" as "amount"', "FROM TEST_DB.TEST_SCHEMA.TEST_TABLE", "SAMPLE (10 ROWS)", "WHERE 1=1", 'ORDER BY "id";'],
+                []
+            ),
+            ( # multiple key columns without double quotes
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ("id as id, region AS region, amount as amount", ['id', 'amount'], ['id', 'region', 'amount']),
+                None,
+                ["ORDER BY id, region;"],
+                []
+            ),
+            ( # multiple key columns with double quotes
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ('"id" as "id", "region" AS "region", "amount" as "amount"', ['id', 'amount'], ['id', 'region', 'amount']),
+                None,
+                ['ORDER BY "id", "region";'],
+                []
+            ),
+            ( # with where clause, no double quotes
+                ['id', 'status'],
+                ['id'],
+                [],
+                [],
+                "WHERE status = 'active'",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "status", "DATA_TYPE": "text"}
+                ],
+                ("id as id, status AS status", ['id'], ['id', 'status']),
+                None,
+                ["WHERE status = 'active'", "ORDER BY id;"],
+                []
+            ),
+            ( # with where clause, with double quotes
+                ['id', 'status'],
+                ['id'],
+                [],
+                [],
+                "WHERE status = 'active'",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "status", "DATA_TYPE": "text"}
+                ],
+                ('"id" as "id", "status" AS "status"', ['id'], ['id', 'status']),
+                None,
+                ["WHERE status = 'active'", 'ORDER BY "id";'],
+                []
+            ),
+            ( # excluded columns, no double quotes
+                ['id', 'name', 'secret'],
+                ['id'],
+                ['secret'],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ("id as id, name AS name", ['id'], ['id', 'name']),
+                None,
+                ["id", "name", "ORDER BY id;"],
+                ["secret"]
+            ),
+            ( # excluded columns, with double quotes
+                ['id', 'name', 'secret'],
+                ['id'],
+                ['secret'],
+                [],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ('"id" as "id", "name" AS "name"', ['id'], ['id', 'name']),
+                None,
+                ['"id"', '"name"', 'ORDER BY "id";'],
+                ["secret"]
+            ),
+            ( # with key filters, no double quotes
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {'id': [1, 2], 'region': ['US', 'EU']},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ("id as id, region AS region, amount as amount", ['id', 'amount'], ['id', 'region', 'amount']),
+                " AND ((ROUND(id, 0),region) in (('1','US'),('2','EU')))",
+                [" AND ((ROUND(id, 0),region) in (('1','US'),('2','EU')))", "ORDER BY id, region;"],
+                []
+            ),
+            ( # with key filters, with double quotes
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {'id': [1, 2], 'region': ['US', 'EU']},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ('"id" as "id", "region" AS "region", "amount" as "amount"', ['id', 'amount'], ['id', 'region', 'amount']),
+                ' AND ((ROUND("id", 0),"region") in ((\'1\',\'US\'),(\'2\',\'EU\')))',
+                [' AND ((ROUND("id", 0),"region") in ((\'1\',\'US\'),(\'2\',\'EU\')))', 'ORDER BY "id", "region";'],
+                []
+            ),
+            ( # dedicated columns, no double quotes
+                ['id', 'name', 'amount', 'description'],
+                ['id'],
+                [],
+                ['id', 'name'],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ("id as id, name AS name", ['id'], ['id', 'name']),
+                None,
+                ["id", "name", "ORDER BY id;"],
+                ["amount", "description"]
+            ),
+            ( # dedicated columns, with double quotes
+                ['id', 'name', 'amount', 'description'],
+                ['id'],
+                [],
+                ['id', 'name'],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ('"id" as "id", "name" AS "name"', ['id'], ['id', 'name']),
+                None,
+                ['"id"', '"name"', 'ORDER BY "id";'],
+                ["amount", "description"]
+            ),
+            ( # custom sample count, no double quotes
+                ['id', 'name'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                50,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ("id as id, name AS name", ['id'], ['id', 'name']),
+                None,
+                ["SAMPLE (50 ROWS)"],
+                []
+            ),
+            ( # custom sample count, with double quotes
+                ['id', 'name'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                50,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"}
+                ],
+                ('"id" as "id", "name" AS "name"', ['id'], ['id', 'name']),
+                None,
+                ["SAMPLE (50 ROWS)"],
+                []
+            ),
+            ( # no key columns (no ORDER BY), no double quotes
+                ['id', 'name', 'amount'],
+                [],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ("id as id, name AS name, amount as amount", ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                ["SELECT id as id, name AS name, amount as amount", "WHERE 1=1 ;"],
+                ["ORDER BY"]
+            ),
+            ( # no key columns (no ORDER BY), with double quotes
+                ['id', 'name', 'amount'],
+                [],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "number"}
+                ],
+                ('"id" as "id", "name" AS "name", "amount" as "amount"', ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                ['SELECT "id" as "id", "name" AS "name", "amount" as "amount"', "WHERE 1=1 ;"],
+                ["ORDER BY"]
+            ),
+            ( # with numeric scale, no double quotes
+                ['id', 'price'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                2,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "price", "DATA_TYPE": "float"}
+                ],
+                ("id as id, CAST(ROUND(price, 2) as decimal(38,2)) as price", ['id', 'price'], ['id', 'price']),
+                None,
+                ["CAST(ROUND(price, 2) as decimal(38,2)) as price"],
+                []
+            ),
+            ( # with numeric scale, with double quotes
+                ['id', 'price'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                2,
+                True,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "price", "DATA_TYPE": "float"}
+                ],
+                ('"id" as "id", CAST(ROUND("price", 2) as decimal(38,2)) as "price"', ['id', 'price'], ['id', 'price']),
+                None,
+                ['CAST(ROUND("price", 2) as decimal(38,2)) as "price"'],
+                []
+            ),
+            ( # special characters with double quotes
+                ['User ID', 'Full Name', 'Email-Address'],
+                ['User ID'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                True,
+                [
+                    {"COLUMN_NAME": "User ID", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "Full Name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "Email-Address", "DATA_TYPE": "text"}
+                ],
+                ('"User ID" as "User ID", "Full Name" AS "Full Name", "Email-Address" AS "Email-Address"', ['User ID'], ['User ID', 'Full Name', 'Email-Address']),
+                None,
+                ['"User ID"', '"Full Name"', '"Email-Address"', 'ORDER BY "User ID";'],
+                []
+            ),
+        ],
+    )
+    def test_create_pandas_df_from_sample(
+        self, snowflake_service, mock_database_object,
+        column_intersections, key_columns, exclude_columns, dedicated_columns,
+        where_clause, key_filters, sample_count, numeric_scale, enclose_quotes,
+        mock_datatypes, mock_column_clause, mock_in_clause,
+        expected_contains, expected_not_in
+    ):
+        """Test create_pandas_df_from_sample with various configurations."""
+        with patch.object(snowflake_service, 'get_data_types_from_object') as mock_get_datatypes, \
+             patch.object(snowflake_service, '_get_column_clause') as mock_get_column, \
+             patch.object(snowflake_service, '_get_in_clause') as mock_get_in, \
+             patch.object(snowflake_service, 'execute_queries') as mock_execute:
+
+            mock_get_datatypes.return_value = mock_datatypes
+            mock_get_column.return_value = mock_column_clause
+            if mock_in_clause:
+                mock_get_in.return_value = mock_in_clause
+            mock_execute.return_value = pd.DataFrame({'id': [1], 'name': ['A']})
+
+            result_list, key_dict, used_columns, sample_query = snowflake_service.create_pandas_df_from_sample(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                key_columns=key_columns,
+                exclude_columns=exclude_columns,
+                dedicated_columns=dedicated_columns,
+                where_clause=where_clause,
+                key_filters=key_filters,
+                sample_count=sample_count,
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=enclose_quotes
+            )
+
+            for expected in expected_contains:
+                assert expected in sample_query
+            for expected in expected_not_in:
+                assert expected not in sample_query

tests/snowflake_service/test_get_checksum_statement.py

@@ -0,0 +1,243 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+@pytest.fixture
+def snowflake_service():
+    """Create a SnowflakeService instance with mocked connection."""
+    mock_params = MagicMock()
+    service = SnowflakeService(mock_params)
+    service.snowflake_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    return DatabaseObject(
+        object_identifier="TestDB.dbo.TestTable",
+        object_type=DatabaseObjectType.TABLE
+    )
+
+
+class TestGetChecksumStatementParametrized:
+    """Parametrized tests for _get_checksum_statement method."""
+
+    @pytest.mark.parametrize(
+        "columns,datatype,exclude_columns,where_clause,numeric_scale," \
+        "enclose_quotes,bool_cast_before_sum,expected_contains,expected_not_in",
+        [
+            ( # numeric with scale, no quotes, no cast before sum
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "number"}],
+                [],
+                "",
+                2,
+                False,
+                False,
+                ["CAST(ROUND(SUM(Amount), 2) AS DECIMAL(38, 2))", 'AS "SUM_Amount"', "FROM TestDB.dbo.TestTable"],
+                []
+            ),
+            ( # numeric with scale, with quotes
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "number"}],
+                [],
+                "",
+                2,
+                True,
+                False,
+                ['CAST(ROUND(SUM("Amount"), 2) AS DECIMAL(38, 2))', 'AS "SUM_Amount"', "FROM TestDB.dbo.TestTable"],
+                []
+            ),
+            ( # numeric without scale, no quotes
+                ["AMOUNT"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "number"}],
+                [],
+                "",
+                None,
+                False,
+                False,
+                ["CAST(SUM(AMOUNT) AS DECIMAL(38))", 'AS "SUM_AMOUNT"'],
+                ["DECIMAL(38,"]
+            ),
+            ( # string column, no quotes
+                ["NAME"],
+                [{"COLUMN_NAME": "NAME", "DATA_TYPE": "text"}],
+                [],
+                "",
+                None,
+                False,
+                False,
+                ["COUNT(DISTINCT LOWER(NAME))", 'AS "COUNTDISTINCT_NAME"'],
+                []
+            ),
+            ( # string column with special char, with quotes
+                ["NAME/"],
+                [{"COLUMN_NAME": "NAME/", "DATA_TYPE": "text"}],
+                [],
+                "",
+                None,
+                True,
+                False,
+                ['COUNT(DISTINCT LOWER("NAME/"))', 'AS "COUNTDISTINCT_NAME/"'],
+                []
+            ),
+            ( # boolean column, no quotes
+                ["IsActive"],
+                [{"COLUMN_NAME": "IsActive", "DATA_TYPE": "boolean"}],
+                [],
+                "",
+                None,
+                False,
+                False,
+                ["COUNT(CASE WHEN IsActive = 1", "COUNT(CASE WHEN IsActive = 0", 'AS "AGGREGATEBOOLEAN_IsActive"'],
+                []
+            ),
+            ( # boolean column, with quotes
+                ["IsActive"],
+                [{"COLUMN_NAME": "IsActive", "DATA_TYPE": "boolean"}],
+                [],
+                "",
+                None,
+                True,
+                False,
+                ['COUNT(CASE WHEN "IsActive" = 1', 'COUNT(CASE WHEN "IsActive" = 0', 'AS "AGGREGATEBOOLEAN_IsActive"'],
+                []
+            ),
+            ( # binary column
+                ["BINARYDATA"],
+                [{"COLUMN_NAME": "BINARYDATA", "DATA_TYPE": "binary"}],
+                [],
+                "",
+                None,
+                False,
+                False,
+                ['TRY_CONVERT(VARCHAR,BINARYDATA)', "COUNT(DISTINCT LOWER("],
+                []
+            ),
+            ( # with where clause
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "number"}],
+                [],
+                "WHERE Amount > 100",
+                None,
+                False,
+                False,
+                ["SUM(Amount)", "WHERE Amount > 100"],
+                []
+            ),
+            ( # excluded columns
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "number"}],
+                ["Price"],
+                "",
+                None,
+                False,
+                False,
+                ["Amount"],
+                ["Price"]
+            ),
+            ( # excluded columns with quotes
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "number"}],
+                ["Price"],
+                "",
+                None,
+                True,
+                False,
+                ['"Amount"'],
+                ['"Price"']
+            ),
+            ( # cast before sum, no quotes
+                ["AMOUNT"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "number"}],
+                [],
+                "",
+                2,
+                False,
+                True,
+                ["ROUND(SUM(CAST(AMOUNT AS DECIMAL(38, 2))), 2)", 'AS "SUM_AMOUNT"'],
+                []
+            ),
+            ( # cast before sum, with quotes
+                ["AMOUNT"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "number"}],
+                [],
+                "",
+                2,
+                True,
+                True,
+                ['ROUND(SUM(CAST("AMOUNT" AS DECIMAL(38, 2))), 2)', 'AS "SUM_AMOUNT"'],
+                []
+            ),
+            ( # multiple columns mixed types
+                ["Amount", "Name/", "ISACTIVE"],
+                [
+                    {"COLUMN_NAME": "Amount", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "Name/", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "ISACTIVE", "DATA_TYPE": "boolean"}
+                ],
+                [],
+                "",
+                2,
+                True,
+                False,
+                ['SUM("Amount")', 'COUNT(DISTINCT LOWER("Name/"))', '"AGGREGATEBOOLEAN_ISACTIVE"'],
+                []
+            ),
+            ( # date column
+                ["CreatedDate"],
+                [{"COLUMN_NAME": "CreatedDate", "DATA_TYPE": "timestamp_ntz"}],
+                [],
+                "",
+                None,
+                False,
+                False,
+                ["COUNT(DISTINCT LOWER(CreatedDate))", 'AS "COUNTDISTINCT_CreatedDate"'],
+                []
+            ),
+            ( # special characters with quotes
+                ["/ISDFPS/OBJNR", "MANDT"],
+                [
+                    {"COLUMN_NAME": "/ISDFPS/OBJNR", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "MANDT", "DATA_TYPE": "number"}
+                ],
+                [],
+                "",
+                None,
+                True,
+                False,
+                ['"/ISDFPS/OBJNR"', '"MANDT"', 'AS "COUNTDISTINCT_/ISDFPS/OBJNR"', 'AS "SUM_MANDT"'],
+                []
+            ),
+        ],
+    )
+    def test_get_checksum_statement(
+        self, snowflake_service, mock_database_object,
+        columns, datatype, exclude_columns, where_clause,
+        numeric_scale, enclose_quotes, bool_cast_before_sum,
+        expected_contains, expected_not_in
+    ):
+        """Test checksum statement with various configurations."""
+        snowflake_service.get_data_types_from_object = MagicMock(return_value=datatype)
+
+        result = snowflake_service._get_checksum_statement(
+            object=mock_database_object,
+            column_intersections=columns,
+            exclude_columns=exclude_columns,
+            where_clause=where_clause,
+            numeric_scale=numeric_scale,
+            enclose_column_by_double_quotes=enclose_quotes,
+            bool_cast_before_sum=bool_cast_before_sum
+        )
+
+        for expected in expected_contains:
+            assert expected in result
+        for expected in expected_not_in:
+            assert expected not in result
+        if where_clause:
+            assert where_clause in result