icsDataValidation 1.0.430__py3-none-any.whl → 1.0.438__py3-none-any.whl

tests/snowflake_service/test_get_column_clause.py

@@ -0,0 +1,305 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+@pytest.fixture
+def snowflake_service():
+    """Fixture for SnowflakeService with mocked connection."""
+    mock_params = MagicMock()
+    service = SnowflakeService(mock_params)
+    return service
+
+class TestGetColumnClauseParametrized:
+    """Parametrized tests for _get_column_clause method."""
+
+    @pytest.mark.parametrize(
+        "column_list,columns_datatype,numeric_scale,key_columns," \
+        "enclose_quotes,expected_clause,expected_numeric,expected_used",
+        [
+            # Numeric with scale
+            (
+                ['price'],
+                [{"COLUMN_NAME": "price", "DATA_TYPE": "float"}],
+                2,
+                [],
+                False,
+                "CAST(ROUND(price, 2) as decimal(38,2)) as price",
+                ['price'],
+                ['price']
+            ),
+            # Numeric without scale
+            (
+                ['price'],
+                [{"COLUMN_NAME": "price", "DATA_TYPE": "float"}],
+                None,
+                [],
+                False,
+                "price as price",
+                ['price'],
+                ['price']
+            ),
+            # String column
+            (
+                ['name'],
+                [{"COLUMN_NAME": "name", "DATA_TYPE": "text"}],
+                None,
+                [],
+                False,
+                "name AS name",
+                [],
+                ['name']
+            ),
+            # Binary column
+            (
+                ['binary_data'],
+                [{"COLUMN_NAME": "binary_data", "DATA_TYPE": "binary"}],
+                None,
+                [],
+                False,
+                "binary_data",
+                [],
+                ['binary_data']
+            ),
+            # Boolean column
+            (
+                ['is_active'],
+                [{"COLUMN_NAME": "is_active", "DATA_TYPE": "boolean"}],
+                None,
+                [],
+                False,
+                "is_active",
+                [],
+                ['is_active']
+            ),
+            # String with double quotes
+            (
+                ['name'],
+                [{"COLUMN_NAME": "name", "DATA_TYPE": "text"}],
+                None,
+                [],
+                True,
+                '"name" AS "name"',
+                [],
+                ['name']
+            ),
+            # Numeric with double quotes and scale
+            (
+                ['amount'],
+                [{"COLUMN_NAME": "amount", "DATA_TYPE": "number"}],
+                2,
+                [],
+                True,
+                'CAST(ROUND("amount", 2) as decimal(38,2)) as "amount"',
+                ['amount'],
+                ['amount']
+            ),
+            # Column with spaces (special characters)
+            (
+                ['Column With Spaces'],
+                [{"COLUMN_NAME": "Column With Spaces", "DATA_TYPE": "text"}],
+                None,
+                [],
+                True,
+                '"Column With Spaces" AS "Column With Spaces"',
+                [],
+                ['Column With Spaces']
+            ),
+            # High precision numeric
+            (
+                ['precise_value'],
+                [{"COLUMN_NAME": "precise_value", "DATA_TYPE": "number"}],
+                10,
+                [],
+                False,
+                "CAST(ROUND(precise_value, 10) as decimal(38,10)) as precise_value",
+                ['precise_value'],
+                ['precise_value']
+            ),
+            # With zero scale
+            (
+                ['count'],
+                [{"COLUMN_NAME": "count", "DATA_TYPE": "number"}],
+                0,
+                [],
+                False,
+                "count as count",
+                ['count'],
+                ['count']
+            ),
+            # empty column list
+            (
+                [],
+                [],
+                0,
+                [],
+                False,
+                "",
+                [],
+                []
+            ),
+        ],
+    )
+    def test_single_column_variations(
+        self, snowflake_service, column_list, columns_datatype, numeric_scale,
+        key_columns, enclose_quotes, expected_clause, expected_numeric, expected_used
+    ):
+        """Test various single column scenarios."""
+        column_clause, numeric_columns, used_columns = snowflake_service._get_column_clause(
+            column_list, columns_datatype, numeric_scale, key_columns, enclose_quotes
+        )
+
+        assert expected_clause in column_clause
+        assert numeric_columns == expected_numeric
+        assert used_columns == expected_used
+
+    @pytest.mark.parametrize(
+        "column_list,columns_datatype,key_columns,expected_in_clause,expected_not_in_clause,expected_used_count",
+        [
+            # Date excluded by default
+            (
+                ['created_date'],
+                [{"COLUMN_NAME": "created_date", "DATA_TYPE": "date"}],
+                [],
+                [],
+                ['created_date'],
+                0
+            ),
+            # Timestamp excluded by default
+            (
+                ['updated_at'],
+                [{"COLUMN_NAME": "updated_at", "DATA_TYPE": "timestamp_ntz"}],
+                [],
+                [],
+                ['updated_at'],
+                0
+            ),
+            # Time excluded by default
+            (
+                ['event_time'],
+                [{"COLUMN_NAME": "event_time", "DATA_TYPE": "time"}],
+                [],
+                [],
+                ['event_time'],
+                0
+            ),
+            # Date included as key column
+            (
+                ['created_date'],
+                [{"COLUMN_NAME": "created_date", "DATA_TYPE": "date"}],
+                ['created_date'],
+                ['created_date'],
+                [],
+                1
+            ),
+            # All Snowflake datetime types excluded
+            (
+                ['col_date', 'col_time', 'col_ts_ntz', 'col_ts_tz', 'col_ts_ltz'],
+                [
+                    {"COLUMN_NAME": "col_date", "DATA_TYPE": "date"},
+                    {"COLUMN_NAME": "col_time", "DATA_TYPE": "time"},
+                    {"COLUMN_NAME": "col_ts_ntz", "DATA_TYPE": "timestamp_ntz"},
+                    {"COLUMN_NAME": "col_ts_tz", "DATA_TYPE": "timestamp_tz"},
+                    {"COLUMN_NAME": "col_ts_ltz", "DATA_TYPE": "timestamp_ltz"}
+                ],
+                [],
+                [],
+                ['col_date', 'col_time', 'col_ts_ntz', 'col_ts_tz', 'col_ts_ltz'],
+                0
+            ),
+            # All datetime types included as key columns
+            (
+                ['col_date', 'col_time', 'col_ts_ntz', 'col_ts_tz', 'col_ts_ltz'],
+                [
+                    {"COLUMN_NAME": "col_date", "DATA_TYPE": "date"},
+                    {"COLUMN_NAME": "col_time", "DATA_TYPE": "time"},
+                    {"COLUMN_NAME": "col_ts_ntz", "DATA_TYPE": "timestamp_ntz"},
+                    {"COLUMN_NAME": "col_ts_tz", "DATA_TYPE": "timestamp_tz"},
+                    {"COLUMN_NAME": "col_ts_ltz", "DATA_TYPE": "timestamp_ltz"}
+                ],
+                ['col_date', 'col_time', 'col_ts_ntz', 'col_ts_tz', 'col_ts_ltz'],
+                ['col_date', 'col_time', 'col_ts_ntz', 'col_ts_tz', 'col_ts_ltz'],
+                [],
+                5
+            ),
+        ],
+    )
+    def test_date_time_column_handling(
+        self, snowflake_service, column_list, columns_datatype, key_columns,
+        expected_in_clause, expected_not_in_clause, expected_used_count
+    ):
+        """Test date/time column inclusion/exclusion logic."""
+        column_clause, numeric_columns, used_columns = snowflake_service._get_column_clause(
+            column_list, columns_datatype, None, key_columns
+        )
+
+        for col in expected_in_clause:
+            assert col in column_clause
+        for col in expected_not_in_clause:
+            assert col not in column_clause
+        assert len(used_columns) == expected_used_count
+
+    @pytest.mark.parametrize(
+        "column_list,columns_datatype,numeric_scale,expected_contains",
+        [
+            # Multiple mixed types
+            (
+                ['id', 'name', 'amount'],
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "float"}
+                ],
+                2,
+                ["CAST(ROUND(id, 2) as decimal(38,2)) as id", "name AS name", "CAST(ROUND(amount, 2) as decimal(38,2)) as amount"]
+            ),
+            # Multiple dates with one key column
+            (
+                ['created_date', 'updated_date', 'deleted_date'],
+                [
+                    {"COLUMN_NAME": "created_date", "DATA_TYPE": "date"},
+                    {"COLUMN_NAME": "updated_date", "DATA_TYPE": "timestamp_ltz"},
+                    {"COLUMN_NAME": "deleted_date", "DATA_TYPE": "timestamp_tz"}
+                ],
+                None,
+                ["created_date"]
+            ),
+        ],
+    )
+    def test_multiple_columns_complex(
+        self, snowflake_service, column_list, columns_datatype, numeric_scale, expected_contains
+    ):
+        """Test complex scenarios with multiple columns."""
+        # For the dates test, only created_date is a key column
+        key_columns = ['created_date'] if 'created_date' in column_list else []
+
+        column_clause, numeric_columns, used_columns = snowflake_service._get_column_clause(
+            column_list, columns_datatype, numeric_scale, key_columns
+        )
+
+        for expected in expected_contains:
+            assert expected in column_clause
+
+    def test_mixed_columns_with_date_excluded_and_key_included(self, snowflake_service):
+        """Test mixed scenario with some date columns excluded and others included as key."""
+        column_list = ['id', 'created_date', 'amount', 'updated_date']
+        columns_datatype = [
+            {"COLUMN_NAME": "id", "DATA_TYPE": "number"},
+            {"COLUMN_NAME": "created_date", "DATA_TYPE": "date"},
+            {"COLUMN_NAME": "amount", "DATA_TYPE": "float"},
+            {"COLUMN_NAME": "updated_date", "DATA_TYPE": "timestamp_ntz"}
+        ]
+        key_columns = ['id', 'created_date']
+
+        column_clause, numeric_columns, used_columns = snowflake_service._get_column_clause(
+            column_list, columns_datatype, 2, key_columns
+        )
+
+        assert "CAST(ROUND(id, 2) as decimal(38,2)) as id" in column_clause
+        assert "created_date" in column_clause
+        assert "CAST(ROUND(amount, 2) as decimal(38,2)) as amount" in column_clause
+        assert "updated_date" not in column_clause
+        assert numeric_columns == ['id', 'amount']
+        assert used_columns == ['id', 'created_date', 'amount']

tests/snowflake_service/test_get_countnulls_statement.py

@@ -0,0 +1,128 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+@pytest.fixture
+def snowflake_service():
+    """Create a SnowflakeService instance with mocked connection."""
+    mock_params = MagicMock()
+    service = SnowflakeService(mock_params)
+    service.snowflake_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    return DatabaseObject(
+        object_identifier="TestDB.dbo.TestTable",
+        object_type=DatabaseObjectType.TABLE
+    )
+
+class TestGetCountnullsStatementParametrized:
+    """Parametrized tests for _get_countnulls_statement method."""
+
+    @pytest.mark.parametrize(
+        "columns,exclude_columns,where_clause," \
+        "enclose_quotes,expected_contains,expected_not_in",
+        [
+         ( # no double quotes, with where clause, exclude one column
+                ["Amount", "Name", "IsActive"],
+                ["IsActive"],
+                'Amount > 0',
+                False,
+                [
+                    "SUM(CASE WHEN Amount IS NULL THEN 1 ELSE 0 END)",
+                    'AS "COUNTNULLS_Amount"',
+                    "SUM(CASE WHEN Name IS NULL THEN 1 ELSE 0 END)",
+                    'AS "COUNTNULLS_Name"',
+
+                ],
+                ["SUM(CASE WHEN IsActive IS NULL THEN 1 ELSE 0 END)",
+                 'AS "COUNTNULLS_IsActive"']
+        ),
+        ( # countnulls statement generation for a single column, non case-sensitive
+                ["Amount"],
+                [],
+                "",
+                False,
+                [
+                    "SUM(CASE WHEN Amount IS NULL THEN 1 ELSE 0 END)",
+                    'AS "COUNTNULLS_Amount"',
+                ],
+                ['"Amount']
+        ),
+        ( # countnulls statement generation for a single column, non case-sensitive
+                ["Amount"],
+                [],
+                "",
+                True,
+                [
+                    "SUM(CASE WHEN \"Amount\" IS NULL THEN 1 ELSE 0 END)",
+                    'AS "COUNTNULLS_Amount"',
+                ],
+                ['"AMOUNT']
+        ),
+        ( # countnulls single column, non case-sensitive, where clause
+                ["Amount"],
+                [],
+                "WHERE Amount > 100",
+                False,
+                [
+                    "SUM(CASE WHEN Amount IS NULL THEN 1 ELSE 0 END)",
+                    'AS "COUNTNULLS_Amount"',
+                    'WHERE Amount > 100'
+                ],
+                ['"Amount']
+        ),
+        (#  excluded columns
+                ["AMOUNT", "PRICE", "QUANTITY"],
+                ["PRICE"],
+                "",
+                False,
+                [
+                    "AMOUNT",
+                    "QUANTITY"
+                ],
+                ['PRICE']
+        ),
+        (#  special characters and column enclosing
+                ["/ISDFPS/OBJNR", "MANDT"],
+                ["PRICE"],
+                "",
+                True,
+                [
+                    '"/ISDFPS/OBJNR"',
+                    'AS "COUNTNULLS_/ISDFPS/OBJNR"',
+                    '"MANDT"',
+                    'AS "COUNTNULLS_MANDT"'
+                ],
+                ['PRICE']
+        )
+        ]
+    )
+    def test_get_countnulls_statement(
+        self, snowflake_service, mock_database_object,
+        columns, exclude_columns, where_clause,
+        enclose_quotes,
+        expected_contains, expected_not_in
+    ):
+        """Test countnulls statement with special characters and double quotes."""
+        result = snowflake_service._get_countnulls_statement(
+            object=mock_database_object,
+            column_intersections=columns,
+            enclose_column_by_double_quotes=enclose_quotes,
+            exclude_columns=exclude_columns,
+            where_clause=where_clause
+        )
+
+        for expected in expected_contains:
+            assert expected in result
+        for expected in expected_not_in:
+            assert expected not in result
+        if where_clause is not None:
+            assert where_clause in result

tests/snowflake_service/test_get_in_clause.py

@@ -0,0 +1,66 @@
+import pytest
+
+from icsDataValidation.services.database_services.snowflake_service import SnowflakeService
+
+
+class TestInClauseVariations:
+    """Test various IN clause generation scenarios using parametrization."""
+
+    @pytest.mark.parametrize(
+        "key_filters,numeric_columns,numeric_scale,enclose_column_by_double_quotes," \
+        "expected_contains",
+        [
+            ( # numeric column with rounding
+                {"price": [10.5, 20.3], "quantity": [5, 10]},
+                ["price"],
+                2,
+                False,
+                ["ROUND(price, 2)", "quantity", "('10.5','5'),('20.3','10')"],
+            ),
+            ( # single row with non numeric columns
+                {"col1": ["value1"], "col2": ["value2"]},
+                [],
+                None,
+                False,
+                [" AND ((col1,col2) in (('value1','value2')))", ]
+            ),
+            ( # multiple rows with tuples and numeric column rounding
+                {"id": [1, 2, 3], "name": ["a", "b", "c"]},
+                ["id"],
+                2,
+                False,
+                [" AND ((ROUND(id, 2),name) in (('1','a'),('2','b'),('3','c')))", ],
+            ),
+            ( # columns with double quotes
+                {"Col1": ["val1"], "Col2": ["val2"]},
+                [],
+                None,
+                True,
+                [' AND (("Col1","Col2")', "('val1','val2')"],
+            ),
+            ( # empty key_filters
+                {},
+                [],
+                None,
+                False,
+                [''],
+            ),
+            ( # special characters in column names with double quotes and numeric rounding
+                {"/ISDFPS/OBJNR": ['000000000012345678', '000000000012345679'], "MANDT": [100, 200]},
+                ["MANDT"],
+                3,
+                True,
+                ['ROUND("MANDT", 3)', '"/ISDFPS/OBJNR"', "('000000000012345678','100'),('000000000012345679','200')"],
+            )
+        ],
+    )
+    def test_in_clause_contains(
+        self, key_filters, numeric_columns, numeric_scale, enclose_column_by_double_quotes, expected_contains
+    ):
+        """Test that result contains expected substrings."""
+        result = SnowflakeService._get_in_clause(
+            key_filters, numeric_columns, numeric_scale, enclose_column_by_double_quotes
+        )
+
+        for expected in expected_contains:
+            assert expected in result

tests/sqlserver_service/test_create_checksums.py

@@ -0,0 +1,153 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
+
+
+@pytest.fixture
+def sqlserver_service():
+    """Create a SQLServerService instance with mocked connection."""
+    connection_params = {
+        'Driver': 'ODBC Driver 18 for SQL Server',
+        'Server': 'localhost',
+        'Port': '1433',
+        'Database': 'testdb',
+        'User': 'sa',
+        'Password': 'password',
+        'Encrypt': True,
+        'TrustServerCertificate': True
+    }
+    service = SQLServerService(connection_params=connection_params)
+    service.sqlserver_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    return DatabaseObject(
+        object_identifier="TestDB.dbo.TestTable",
+        object_type=DatabaseObjectType.TABLE
+    )
+
+
+class TestCreateChecksumsParametrized:
+    """Parametrized tests for SQLServerService.create_checksums."""
+
+    @pytest.mark.parametrize(
+        "column_intersections,where_clause,numeric_scale,execute_behavior," \
+        "expected_columns,expected_errors,expect_retry,expected_execute_calls",
+        [
+            (  # success path
+                ['amount', 'name'],
+                'WHERE amount > 0',
+                2,
+                {
+                    "return_value": [
+                        [{'SUM_AMOUNT': 10, 'COUNTDISTINCT_NAME': 3}],
+                        [{'COUNTNULLS_AMOUNT': 1, 'COUNTNULLS_NAME': 0}]
+                    ]
+                },
+                {
+                    'AMOUNT': ['SUM', 10, 1],
+                    'NAME': ['COUNTDISTINCT', 3, 0]
+                },
+                [],
+                False,
+                1
+            ),
+            (  # arithmetic overflow triggers retry
+                ['amount'],
+                '',
+                None,
+                {
+                    "side_effect": [
+                        Exception('checksum_sql|||Arithmetic overflow error converting numeric to data type numeric'),
+                        [[{'SUM_AMOUNT': 5}], [{'COUNTNULLS_AMOUNT': 0}]]
+                    ]
+                },
+                {
+                    'AMOUNT': ['SUM', 5, 0]
+                },
+                [],
+                True,
+                2
+            ),
+            (  # non-overflow error surfaces in TESTATM_ERRORS
+                ['amount'],
+                '',
+                None,
+                {
+                    "side_effect": Exception('checksum_sql|||Some other error')
+                },
+                {},
+                [['ERROR', 'checksum_sql', 'Some other error']],
+                False,
+                1
+            ),
+        ],
+    )
+    def test_create_checksums(
+        self,
+        sqlserver_service,
+        mock_database_object,
+        column_intersections,
+        where_clause,
+        numeric_scale,
+        execute_behavior,
+        expected_columns,
+        expected_errors,
+        expect_retry,
+        expected_execute_calls
+    ):
+        """Test create_checksums behavior across success, retry, and error scenarios."""
+        sqlserver_service.create_checksum_statement = MagicMock(return_value='checksum_retry_sql')
+
+        with patch.object(sqlserver_service, '_get_checksum_statement', return_value='checksum_sql') as mock_checksum_stmt, \
+             patch.object(sqlserver_service, '_get_countnulls_statement', return_value='countnulls_sql') as mock_countnulls_stmt, \
+             patch.object(sqlserver_service, 'execute_queries') as mock_execute:
+
+            if 'side_effect' in execute_behavior:
+                mock_execute.side_effect = execute_behavior['side_effect']
+            else:
+                mock_execute.return_value = execute_behavior['return_value']
+
+            result = sqlserver_service.create_checksums(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[],
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=False
+            )
+
+            mock_checksum_stmt.assert_called_once_with(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[],
+                numeric_scale=numeric_scale
+            )
+            mock_countnulls_stmt.assert_called_once_with(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                where_clause=where_clause,
+                exclude_columns=[]
+            )
+            assert mock_execute.call_count == expected_execute_calls
+
+            if expect_retry:
+                sqlserver_service.create_checksum_statement.assert_called_once()
+                retry_kwargs = sqlserver_service.create_checksum_statement.call_args.kwargs
+                assert retry_kwargs['bool_cast_before_sum'] is True
+            else:
+                sqlserver_service.create_checksum_statement.assert_not_called()
+
+            for column, expected in expected_columns.items():
+                assert result[column] == expected
+
+            expected_keys = set(expected_columns.keys()) | {'TESTATM_ERRORS'}
+            assert set(result.keys()) == expected_keys
+            assert result['TESTATM_ERRORS'] == expected_errors