icsDataValidation 1.0.430__py3-none-any.whl → 1.0.438__py3-none-any.whl

tests/sqlserver_service/test_create_pandas_df_from_group_by.py

@@ -0,0 +1,427 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from icsDataValidation.core.database_objects import DatabaseObject, DatabaseObjectType
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
+
+
+@pytest.fixture
+def sqlserver_service():
+    """Create a SQLServerService instance with mocked connection."""
+    connection_params = {
+        'Driver': 'ODBC Driver 18 for SQL Server',
+        'Server': 'localhost',
+        'Port': '1433',
+        'Database': 'testdb',
+        'User': 'sa',
+        'Password': 'password',
+        'Encrypt': True,
+        'TrustServerCertificate': True
+    }
+    service = SQLServerService(connection_params=connection_params)
+    service.sqlserver_connection = MagicMock()
+    return service
+
+
+@pytest.fixture
+def mock_database_object():
+    """Create a mock DatabaseObject."""
+    obj = DatabaseObject(
+        object_identifier="TestDB.dbo.TestTable",
+        object_type=DatabaseObjectType.TABLE
+    )
+    return obj
+
+
+class TestCreatePandasDfFromGroupByParametrized:
+    """Parametrized tests for create_pandas_df_from_group_by method."""
+
+    @pytest.mark.parametrize(
+        "column_intersections,group_by_columns,group_by_aggregation_columns," \
+        "group_by_aggregation_type,only_numeric,where_clause,exclude_columns," \
+        "numeric_scale,enclose_quotes,mock_datatypes," \
+        "expected_group_by_cols,expected_in_agg_string,expected_not_in_agg_string," \
+        "expected_grouping_cols_final",
+        [
+            ( # single grouping column, no double quotes
+                ['region', 'amount'],
+                ['region'],
+                ['amount'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [region] ",
+                ["SUM([AMOUNT])"],
+                [],
+                ['region']
+            ),
+            ( # single grouping column, with double quotes (ignored/treated same in SQL Server impl)
+                ['region', 'amount'],
+                ['region'],
+                ['amount'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                True,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [region] ",
+                ["SUM([AMOUNT])"],
+                [],
+                ['region']
+            ),
+            ( # multiple grouping columns
+                ['region', 'department', 'amount'],
+                ['region', 'department'],
+                ['amount'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [region] ,[department] ",
+                ["SUM([AMOUNT])"],
+                [],
+                ['region', 'department']
+            ),
+            ( # grouping column excluded
+                ['region', 'department', 'amount'],
+                ['region', 'department'],
+                ['amount'],
+                'various',
+                False,
+                '',
+                ['department'],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [region] ",
+                ["SUM([AMOUNT])"],
+                [],
+                ['region']
+            ),
+            ( # grouping column not in intersections
+                ['amount'],
+                ['region'],
+                ['amount'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                "",
+                [],
+                [],
+                []
+            ),
+            ( # only_min_max type, numeric columns
+                ['region', 'amount', 'price'],
+                ['region'],
+                ['amount', 'price'],
+                'only_min_max',
+                True,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "PRICE", "DATA_TYPE": "decimal"}
+                ],
+                " [region] ",
+                ["MIN([AMOUNT])", "MAX([AMOUNT])", "MIN([PRICE])", "MAX([PRICE])"],
+                ["SUM(", "COUNTDISTINCT"],
+                ['region']
+            ),
+            ( # only_min_max with numeric_scale
+                ['region', 'AMOUNT'],
+                ['region'],
+                ['AMOUNT'],
+                'only_min_max',
+                True,
+                '',
+                [],
+                2,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "decimal"}],
+                " [region] ",
+                ["CAST(ROUND(MIN([AMOUNT]),2) AS DECIMAL(38,2))", "CAST(ROUND(MAX([AMOUNT]),2) AS DECIMAL(38,2))"],
+                [],
+                ['region']
+            ),
+            ( # various type, numeric only
+                ['REGION', 'AMOUNT'],
+                ['REGION'],
+                ['AMOUNT'],
+                'various',
+                True,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [REGION] ",
+                ["SUM([AMOUNT])"],
+                ["MIN(", "MAX("],
+                ['REGION']
+            ),
+            ( # various type with string columns
+                ['region', 'AMOUNT', 'DESCRIPTION'],
+                ['region'],
+                ['AMOUNT', 'DESCRIPTION'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "DESCRIPTION", "DATA_TYPE": "varchar"}
+                ],
+                " [region] ",
+                ["SUM([AMOUNT])", "COUNT(DISTINCT LOWER([DESCRIPTION]))", "[COUNTDISTINCT_DESCRIPTION]"],
+                [],
+                ['region']
+            ),
+            ( # various type with boolean columns
+                ['REGION', 'AMOUNT', 'IS_ACTIVE'],
+                ['REGION'],
+                ['AMOUNT', 'IS_ACTIVE'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "IS_ACTIVE", "DATA_TYPE": "bit"}
+                ],
+                " [REGION] ",
+                ["SUM([AMOUNT])", "AGGREGATEBOOLEAN_IS_ACTIVE", "CASE WHEN [IS_ACTIVE] = 1"],
+                [],
+                ['REGION']
+            ),
+            ( # various type with binary columns
+                ['REGION', 'BINARY_DATA'],
+                ['REGION'],
+                ['BINARY_DATA'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "BINARY_DATA", "DATA_TYPE": "varbinary"}],
+                " [REGION] ",
+                ["COUNT(DISTINCT LOWER(TRY_CONVERT(VARCHAR,[BINARY_DATA])))", "COUNTDISTINCT_BINARY_DATA"],
+                [],
+                ['REGION']
+            ),
+            ( # various type with datetime columns
+                ['REGION', 'CREATED_DATE'],
+                ['REGION'],
+                ['CREATED_DATE'],
+                'various',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [{"COLUMN_NAME": "CREATED_DATE", "DATA_TYPE": "datetime"}],
+                " [REGION] ",
+                ["COUNT(DISTINCT LOWER([CREATED_DATE]))", "COUNTDISTINCT_CREATED_DATE"],
+                [],
+                ['REGION']
+            ),
+            ( # various_and_min_max type
+                ['REGION', 'AMOUNT', 'DESCRIPTION'],
+                ['REGION'],
+                ['AMOUNT', 'DESCRIPTION'],
+                'various_and_min_max',
+                False,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "DESCRIPTION", "DATA_TYPE": "varchar"}
+                ],
+                " [REGION] ",
+                ["MIN([AMOUNT])", "MAX([AMOUNT])", "SUM([AMOUNT])", "COUNT(DISTINCT LOWER([DESCRIPTION]))"],
+                [],
+                ['REGION']
+            ),
+            ( # aggregation columns 'all'
+                ['REGION', 'AMOUNT', 'PRICE'],
+                ['REGION'],
+                ['all'],
+                'various',
+                True,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "PRICE", "DATA_TYPE": "decimal"}
+                ],
+                " [REGION] ",
+                ["SUM([AMOUNT])", "SUM([PRICE])"],
+                ["SUM([REGION])"],
+                ['REGION']
+            ),
+            ( # aggregation with exclude_columns
+                ['REGION', 'AMOUNT', 'PRICE'],
+                ['REGION'],
+                ['AMOUNT', 'PRICE'],
+                'various',
+                True,
+                '',
+                ['PRICE'],
+                None,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"}],
+                " [REGION] ",
+                ["SUM([AMOUNT])"],
+                ["SUM([PRICE])"],
+                ['REGION']
+            ),
+            ( # numeric_scale with various type
+                ['REGION', 'AMOUNT'],
+                ['REGION'],
+                ['AMOUNT'],
+                'various',
+                True,
+                '',
+                [],
+                3,
+                False,
+                [{"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "decimal"}],
+                " [REGION] ",
+                ["CAST(ROUND(SUM([AMOUNT]), 3) AS DECIMAL(38,3))"],
+                [],
+                ['REGION']
+            ),
+            ( # mixed datatype aggregations
+                ['REGION', 'AMOUNT', 'PRICE', 'NAME', 'IS_ACTIVE', 'CREATED_DATE'],
+                ['REGION'],
+                ['AMOUNT', 'PRICE', 'NAME', 'IS_ACTIVE', 'CREATED_DATE'],
+                'various',
+                False,
+                '',
+                [],
+                2,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "PRICE", "DATA_TYPE": "decimal"},
+                    {"COLUMN_NAME": "NAME", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "IS_ACTIVE", "DATA_TYPE": "bit"},
+                    {"COLUMN_NAME": "CREATED_DATE", "DATA_TYPE": "datetime"}
+                ],
+                " [REGION] ",
+                [
+                    "CAST(ROUND(SUM([AMOUNT]), 2) AS DECIMAL(38,2))",
+                    "CAST(ROUND(SUM([PRICE]), 2) AS DECIMAL(38,2))",
+                    "COUNT(DISTINCT LOWER([NAME]))",
+                    "AGGREGATEBOOLEAN_IS_ACTIVE",
+                    "COUNT(DISTINCT LOWER([CREATED_DATE]))"
+                ],
+                [],
+                ['REGION']
+            ),
+            ( # only_numeric flag excludes string aggregations
+                ['REGION', 'AMOUNT', 'NAME', 'IS_ACTIVE'],
+                ['REGION'],
+                ['AMOUNT', 'NAME', 'IS_ACTIVE'],
+                'various',
+                True,
+                '',
+                [],
+                None,
+                False,
+                [
+                    {"COLUMN_NAME": "AMOUNT", "DATA_TYPE": "int"},
+                    {"COLUMN_NAME": "NAME", "DATA_TYPE": "varchar"},
+                    {"COLUMN_NAME": "IS_ACTIVE", "DATA_TYPE": "bit"}
+                ],
+                " [REGION] ",
+                ["SUM([AMOUNT])"],
+                ["COUNTDISTINCT", "AGGREGATEBOOLEAN"],
+                ['REGION']
+            ),
+            ( # special character column names
+                ['region', '/ISDFPS/amount'],
+                ['region'],
+                ['/ISDFPS/amount'],
+                'various',
+                True,
+                '',
+                [],
+                None,
+                True,
+                [{"COLUMN_NAME": "/ISDFPS/AMOUNT", "DATA_TYPE": "int"}],
+                " [region] ",
+                ["[/ISDFPS/AMOUNT]"],
+                [],
+                ['region']
+            ),
+        ],
+    )
+    def test_create_pandas_df_from_group_by(
+        self, sqlserver_service, mock_database_object,
+        column_intersections, group_by_columns, group_by_aggregation_columns,
+        group_by_aggregation_type, only_numeric, where_clause, exclude_columns,
+        numeric_scale, enclose_quotes, mock_datatypes,
+        expected_group_by_cols, expected_in_agg_string, expected_not_in_agg_string,
+        expected_grouping_cols_final
+    ):
+        """Test create_pandas_df_from_group_by with various configurations."""
+        with patch.object(sqlserver_service, 'get_data_types_from_object') as mock_get_datatypes, \
+             patch.object(sqlserver_service, 'execute_queries') as mock_execute:
+
+            mock_get_datatypes.return_value = mock_datatypes
+            mock_execute.return_value = pd.DataFrame()
+
+            result = sqlserver_service.create_pandas_df_from_group_by(
+                object=mock_database_object,
+                column_intersections=column_intersections,
+                group_by_columns=group_by_columns,
+                group_by_aggregation_columns=group_by_aggregation_columns,
+                group_by_aggregation_type=group_by_aggregation_type,
+                only_numeric=only_numeric,
+                where_clause=where_clause,
+                exclude_columns=exclude_columns,
+                numeric_scale=numeric_scale,
+                enclose_column_by_double_quotes=enclose_quotes
+            )
+
+            _, group_by_query_aggregation_string, group_by_query_columns_string, grouping_columns_final, _ = result
+
+            # Check group_by_query_columns_string
+            assert group_by_query_columns_string == expected_group_by_cols
+
+            # Check grouping_columns_final
+            assert grouping_columns_final == expected_grouping_cols_final
+
+            # Check expected strings in aggregation string
+            for expected in expected_in_agg_string:
+                assert expected in group_by_query_aggregation_string
+
+            # Check strings that should NOT be in aggregation string
+            for expected in expected_not_in_agg_string:
+                assert expected not in group_by_query_aggregation_string