icsDataValidation 1.0.430__py3-none-any.whl → 1.0.439__py3-none-any.whl

tests/sqlserver_service/test_create_pandas_df_from_sample.py

@@ -0,0 +1,286 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
+
+
+@pytest.fixture
+def sqlserver_service():
+    """Create a SQLServerService instance with mocked connection."""
+    connection_params = {
+        'Driver': 'ODBC Driver 18 for SQL Server',
+        'Server': 'localhost',
+        'Port': '1433',
+        'Database': 'testdb',
+        'User': 'sa',
+        'Password': 'password',
+        'Encrypt': True,
+        'TrustServerCertificate': True
+    }
+    service = SQLServerService(connection_params=connection_params)
+    service.sqlserver_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    obj = DatabaseObject(
+        object_identifier="TEST_DB.TEST_SCHEMA.TEST_TABLE",
+        object_type=DatabaseObjectType.TABLE
+    )
+    return obj
+
+
+class TestCreatePandasDfFromSampleParametrized:
+    """Parametrized tests for create_pandas_df_from_sample method."""
+
+    @pytest.mark.parametrize(
+        "column_intersections,key_columns,exclude_columns,dedicated_columns,"
+        "where_clause,key_filters,sample_count,numeric_scale,enclose_quotes,"
+        "mock_datatypes,mock_column_clause,mock_in_clause,expected_contains,expected_not_in",
+        [
+            (  # simple case with key columns
+                ['id', 'name', 'amount'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "int"}
+                ],
+                ("[id] as [id], [name] AS [name], [amount] as [amount]", ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                [
+                    "SELECT TOP (10) [id] as [id], [name] AS [name], [amount] as [amount]",
+                    "FROM TEST_SCHEMA.TEST_TABLE",
+                    "WHERE 1=1",
+                    "ORDER BY [id];"
+                ],
+                []
+            ),
+            (  # multiple key columns
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "int"}
+                ],
+                ("[id] as [id], [region] AS [region], [amount] as [amount]", ['id', 'amount'], ['id', 'region', 'amount']),
+                None,
+                ["ORDER BY [id], [region];"],
+                []
+            ),
+            (  # with where clause
+                ['id', 'status'],
+                ['id'],
+                [],
+                [],
+                "WHERE status = 'active'",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "status", "DATA_TYPE": "varchar"}
+                ],
+                ("[id] as [id], [status] AS [status]", ['id'], ['id', 'status']),
+                None,
+                ["WHERE status = 'active'", "ORDER BY [id];"],
+                []
+            ),
+            (  # excluded columns
+                ['id', 'name', 'secret'],
+                ['id'],
+                ['secret'],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "varchar"}
+                ],
+                ("[id] as [id], [name] AS [name]", ['id'], ['id', 'name']),
+                None,
+                ["[id]", "[name]", "ORDER BY [id];"],
+                ["[secret]"]
+            ),
+            (  # with key filters
+                ['id', 'region', 'amount'],
+                ['id', 'region'],
+                [],
+                [],
+                "",
+                {'id': [1, 2], 'region': ['US', 'EU']},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "region", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "int"}
+                ],
+                ("[id] as [id], [region] AS [region], [amount] as [amount]", ['id', 'amount'], ['id', 'region', 'amount']),
+                " AND (CONCAT(cast(ROUND([id], 0) as numeric(38, 0)), '|' ,[region], '|') in ('1|US|','2|EU|'))",
+                [
+                    "AND (CONCAT(cast(ROUND([id], 0) as numeric(38, 0)), '|' ,[region], '|') in ('1|US|','2|EU|'))",
+                    "ORDER BY [id], [region];"
+                ],
+                []
+            ),
+            (  # dedicated columns
+                ['id', 'name', 'amount', 'description'],
+                ['id'],
+                [],
+                ['id', 'name'],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "varchar"}
+                ],
+                ("[id] as [id], [name] AS [name]", ['id'], ['id', 'name']),
+                None,
+                ["[id]", "[name]", "ORDER BY [id];"],
+                ["[amount]", "[description]"]
+            ),
+            (  # custom sample count
+                ['id', 'name'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                50,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "varchar"}
+                ],
+                ("[id] as [id], [name] AS [name]", ['id'], ['id', 'name']),
+                None,
+                ["SELECT TOP (50)"] ,
+                []
+            ),
+            (  # no key columns (uses ORDER BY NEWID())
+                ['id', 'name', 'amount'],
+                [],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "name", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "amount", "DATA_TYPE": "int"}
+                ],
+                ("[id] as [id], [name] AS [name], [amount] as [amount]", ['id', 'amount'], ['id', 'name', 'amount']),
+                None,
+                ["SELECT TOP (10) [id] as [id], [name] AS [name], [amount] as [amount]", "WHERE 1=1", "ORDER BY NEWID();"],
+                ["ORDER BY ["]
+            ),
+            (  # with numeric scale, no double quotes
+                ['id', 'price'],
+                ['id'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                2,
+                False,
+                [
+                    {"COLUMN_NAME": "id", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "price", "DATA_TYPE": "decimal"}
+                ],
+                ("[id] as [id], CAST(ROUND([price], 2) as decimal(38,2)) as [price]", ['id', 'price'], ['id', 'price']),
+                None,
+                ["CAST(ROUND([price], 2) as decimal(38,2)) as [price]"],
+                []
+            ),
+            (  # special characters
+                ['User ID', 'Full Name', 'Email-Address'],
+                ['User ID'],
+                [],
+                [],
+                "",
+                {},
+                10,
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "User ID", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "Full Name", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "Email-Address", "DATA_TYPE": "varchar"}
+                ],
+                ("[User ID] as [User ID], [Full Name] AS [Full Name], [Email-Address] AS [Email-Address]", ['User ID'], ['User ID', 'Full Name', 'Email-Address']),
+                None,
+                ["[User ID]", "[Full Name]", "[Email-Address]", "ORDER BY [User ID];"],
+                []
+            ),
+        ],
+    )
+    def test_create_pandas_df_from_sample(
+        self, sqlserver_service, mock_database_object,
+        column_intersections, key_columns, exclude_columns, dedicated_columns,
+        where_clause, key_filters, sample_count, numeric_scale, enclose_quotes,
+        mock_datatypes, mock_column_clause, mock_in_clause,
+        expected_contains, expected_not_in
+    ):
+        """Test create_pandas_df_from_sample with various configurations."""
+        with patch.object(sqlserver_service, 'get_data_types_from_object') as mock_get_datatypes, \
+             patch.object(sqlserver_service, '_get_column_clause') as mock_get_column, \
+             patch.object(sqlserver_service, '_get_in_clause') as mock_get_in, \
+             patch.object(sqlserver_service, 'execute_queries') as mock_execute:
+
+            mock_get_datatypes.return_value = mock_datatypes
+            mock_get_column.return_value = mock_column_clause
+            if mock_in_clause:
+                mock_get_in.return_value = mock_in_clause
+            mock_execute.return_value = pd.DataFrame({'id': [1], 'name': ['A'], 'region': ['US'], 'User ID': [123]})
+
+            result_list, key_dict, used_columns, sample_query = sqlserver_service.create_pandas_df_from_sample(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                key_columns=key_columns,
+                exclude_columns=exclude_columns,
+                dedicated_columns=dedicated_columns,
+                where_clause=where_clause,
+                key_filters=key_filters,
+                sample_count=sample_count,
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=enclose_quotes
+            )
+
+            for expected in expected_contains:
+                assert expected in sample_query
+            for expected in expected_not_in:
+                assert expected not in sample_query

tests/sqlserver_service/test_get_checksum_statement.py

@@ -0,0 +1,160 @@
+from unittest.mock import MagicMock, Mock
+
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
+
+
+@pytest.fixture
+def sqlserver_service():
+    """Create a SQLServerService instance with mocked connection."""
+    connection_params = {
+        'Driver': 'ODBC Driver 18 for SQL Server',
+        'Server': 'localhost',
+        'Port': '1433',
+        'Database': 'testdb',
+        'User': 'sa',
+        'Password': 'password',
+        'Encrypt': True,
+        'TrustServerCertificate': True
+    }
+    service = SQLServerService(connection_params=connection_params)
+    service.sqlserver_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    obj = Mock(spec=DatabaseObject)
+    obj.database = "TestDB"
+    obj.schema = "dbo"
+    obj.name = "TestTable"
+    obj.type = "table"
+    return obj
+
+
+class TestGetChecksumStatementParametrized:
+    """Parametrized tests for _get_checksum_statement method."""
+
+    @pytest.mark.parametrize(
+        "columns,mock_datatypes,exclude_columns,where_clause,numeric_scale," \
+        "expected_contains,expected_not_in",
+        [
+            ( # numeric column with scale
+                ["Amount"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "decimal"}],
+                [],
+                "",
+                2,
+                ["SUM([AMOUNT])", "DECIMAL(38, 2)", "AS [SUM_AMOUNT]", "FROM dbo.TestTable"],
+                []
+            ),
+            ( # numeric column without scale
+                ["Amount"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                [],
+                "",
+                None,
+                ["SUM([AMOUNT])", "AS [SUM_AMOUNT]"],
+                ["DECIMAL(38,"]
+            ),
+            ( # string column
+                ["Name"],
+                [{"COLUMN_NAME": "NAME", "DATA_TYPE": "varchar"}],
+                [],
+                "",
+                None,
+                ["COUNT(DISTINCT LOWER([NAME]))", "AS [COUNTDISTINCT_NAME]"],
+                []
+            ),
+            ( # boolean column
+                ["IsActive"],
+                [{"COLUMN_NAME": "ISACTIVE", "DATA_TYPE": "bit"}],
+                [],
+                "",
+                None,
+                ["COUNT(CASE WHEN [ISACTIVE] = 1", "COUNT(CASE WHEN [ISACTIVE] = 0", "AS [AGGREGATEBOOLEAN_ISACTIVE]"],
+                []
+            ),
+            ( # with where clause
+                ["Amount"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                [],
+                "WHERE Amount > 100",
+                None,
+                ["WHERE Amount > 100"],
+                []
+            ),
+            ( # exclude columns
+                ["Amount", "Price"],
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                ["Price"],
+                "",
+                None,
+                ["AMOUNT"],
+                ["PRICE"]
+            ),
+            ( # multiple columns mixed types
+                ["Amount", "Name", "IsActive"],
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "decimal"},
+                    {"COLUMN_NAME": "NAME", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "ISACTIVE", "DATA_TYPE": "bit"}
+                ],
+                [],
+                "",
+                2,
+                ["SUM([AMOUNT])", "COUNT(DISTINCT LOWER([NAME]))", "AGGREGATEBOOLEAN_ISACTIVE"],
+                []
+            ),
+            ( # binary column
+                ["BinaryData"],
+                [{"COLUMN_NAME": "BINARYDATA", "DATA_TYPE": "varbinary"}],
+                [],
+                "",
+                None,
+                ["TRY_CONVERT(VARCHAR,[BINARYDATA])", "COUNT(DISTINCT LOWER("],
+                []
+            ),
+            ( # date column
+                ["CreatedDate"],
+                [{"COLUMN_NAME": "CREATEDDATE", "DATA_TYPE": "datetime"}],
+                [],
+                "",
+                None,
+                ["COUNT(DISTINCT LOWER([CREATEDDATE]))", "AS [COUNTDISTINCT_CREATEDDATE]"],
+                []
+            ),
+            ( # special characters in column names
+                ["/ISDFPS/OBJNR"],
+                [{"COLUMN_NAME": "/ISDFPS/OBJNR", "DATA_TYPE": "varchar"}],
+                [],
+                "",
+                None,
+                ["[/ISDFPS/OBJNR]", "AS [COUNTDISTINCT_/ISDFPS/OBJNR]"],
+                []
+            ),
+        ],
+    )
+    def test_get_checksum_statement(
+        self, sqlserver_service, mock_database_object,
+        columns, mock_datatypes, exclude_columns, where_clause, numeric_scale,
+        expected_contains, expected_not_in
+    ):
+        """Test checksum statement with various configurations."""
+        sqlserver_service.get_data_types_from_object = MagicMock(return_value=mock_datatypes)
+
+        result = sqlserver_service._get_checksum_statement(
+            object=mock_database_object,
+            column_intersections=columns,
+            exclude_columns=exclude_columns,
+            where_clause=where_clause,
+            numeric_scale=numeric_scale
+        )
+
+        for expected in expected_contains:
+            assert expected in result
+        for expected in expected_not_in:
+            assert expected not in result

tests/sqlserver_service/test_get_column_clause.py

@@ -0,0 +1,182 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
+
+
+@pytest.fixture
+def sqlserver_service():
+    """Create a SQLServerService instance with mocked connection."""
+    connection_params = {
+        'Driver': 'ODBC Driver 18 for SQL Server',
+        'Server': 'localhost',
+        'Port': '1433',
+        'Database': 'testdb',
+        'User': 'sa',
+        'Password': 'password',
+        'Encrypt': True,
+        'TrustServerCertificate': True
+    }
+    service = SQLServerService(connection_params=connection_params)
+    service.sqlserver_connection = MagicMock()
+    return service
+
+
+class TestGetColumnClauseParametrized:
+    """Parametrized tests for _get_column_clause method."""
+
+    @pytest.mark.parametrize(
+        "column_list,columns_datatype,numeric_scale,key_columns," \
+        "expected_in_clause,expected_numeric,expected_used",
+        [
+            ( # numeric with scale
+                ["Amount"],
+                [{"COLUMN_NAME": "Amount", "DATA_TYPE": "decimal"}],
+                2,
+                [],
+                ["CAST(ROUND([Amount], 2) as decimal(38,2)) as [Amount]"],
+                ["Amount"],
+                ["Amount"]
+            ),
+            ( # numeric without scale
+                ["Price"],
+                [{"COLUMN_NAME": "Price", "DATA_TYPE": "int"}],
+                None,
+                [],
+                ["[Price] as [Price]"],
+                ["Price"],
+                ["Price"]
+            ),
+            ( # string column
+                ["Name"],
+                [{"COLUMN_NAME": "Name", "DATA_TYPE": "varchar"}],
+                2,
+                [],
+                ["[Name] AS [Name]"],
+                [],
+                ["Name"]
+            ),
+            ( # date excluded by default
+                ["CreatedDate"],
+                [{"COLUMN_NAME": "CreatedDate", "DATA_TYPE": "datetime"}],
+                2,
+                [],
+                [],
+                [],
+                []
+            ),
+            ( # date as key column
+                ["CreatedDate"],
+                [{"COLUMN_NAME": "CreatedDate", "DATA_TYPE": "datetime"}],
+                2,
+                ["CreatedDate"],
+                ["CreatedDate"],
+                [],
+                ["CreatedDate"]
+            ),
+            ( # multiple mixed types
+                ["Amount", "Name", "IsActive"],
+                [
+                    {"COLUMN_NAME": "Amount", "DATA_TYPE": "decimal"},
+                    {"COLUMN_NAME": "Name", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "IsActive", "DATA_TYPE": "bit"}
+                ],
+                2,
+                [],
+                ["CAST(ROUND([Amount], 2)", "[Name] AS [Name]", "IsActive"],
+                ["Amount"],
+                ["Amount", "Name", "IsActive"]
+            ),
+            ( # special characters in column names
+                ["/ISDFPS/OBJNR", "MANDT"],
+                [
+                    {"COLUMN_NAME": "/ISDFPS/OBJNR", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "MANDT", "DATA_TYPE": "varchar"}
+                ],
+                2,
+                [],
+                ["[/ISDFPS/OBJNR]", "[MANDT]"],
+                [],
+                ["/ISDFPS/OBJNR", "MANDT"]
+            ),
+            ( # binary column
+                ["BinaryData"],
+                [{"COLUMN_NAME": "BinaryData", "DATA_TYPE": "varbinary"}],
+                2,
+                [],
+                ["BinaryData"],
+                [],
+                ["BinaryData"]
+            ),
+            ( # empty column list
+                [],
+                [],
+                2,
+                [],
+                [],
+                [],
+                []
+            ),
+            ( # all numeric columns with scale
+                ["Amount", "Price", "Quantity"],
+                [
+                    {"COLUMN_NAME": "Amount", "DATA_TYPE": "decimal"},
+                    {"COLUMN_NAME": "Price", "DATA_TYPE": "money"},
+                    {"COLUMN_NAME": "Quantity", "DATA_TYPE": "int"}
+                ],
+                3,
+                [],
+                ["decimal(38,3)"],
+                ["Amount", "Price", "Quantity"],
+                ["Amount", "Price", "Quantity"]
+            ),
+            ( # numeric with zero scale
+                ["Count"],
+                [{"COLUMN_NAME": "Count", "DATA_TYPE": "int"}],
+                0,
+                [],
+                ["[Count] as [Count]"],
+                ["Count"],
+                ["Count"]
+            ),
+            ( # text and ntext columns
+                ["Description", "Notes"],
+                [
+                    {"COLUMN_NAME": "Description", "DATA_TYPE": "text"},
+                    {"COLUMN_NAME": "Notes", "DATA_TYPE": "ntext"}
+                ],
+                2,
+                [],
+                ["[Description] AS [Description]", "[Notes] AS [Notes]"],
+                [],
+                ["Description", "Notes"]
+            ),
+            ( # multiple date columns with one as key
+                ["CreatedDate", "ModifiedDate", "Id"],
+                [
+                    {"COLUMN_NAME": "CreatedDate", "DATA_TYPE": "datetime"},
+                    {"COLUMN_NAME": "ModifiedDate", "DATA_TYPE": "datetime2"},
+                    {"COLUMN_NAME": "Id", "DATA_TYPE": "int"}
+                ],
+                2,
+                ["CreatedDate"],
+                ["CreatedDate", "Id"],
+                ["Id"],
+                ["CreatedDate", "Id"]
+            ),
+        ],
+    )
+    def test_get_column_clause(
+        self, sqlserver_service, column_list, columns_datatype, numeric_scale,
+        key_columns, expected_in_clause, expected_numeric, expected_used
+    ):
+        """Test column clause with various configurations."""
+        column_clause, numeric_columns, used_columns = sqlserver_service._get_column_clause(
+            column_list, columns_datatype, numeric_scale, key_columns
+        )
+
+        for expected in expected_in_clause:
+            assert expected in column_clause
+        assert set(numeric_columns) == set(expected_numeric)
+        assert set(used_columns) == set(expected_used)