icsDataValidation 1.0.371__py3-none-any.whl → 1.0.415__py3-none-any.whl

icsDataValidation/services/database_services/teradata_service.py

@@ -110,7 +110,7 @@ class TeradataService(object):
         try:
 
             row_count = self.execute_queries(query_get_row_count).fetchall()[0][0]
-        
+
         except Exception as err:
             error_list.append(str(err))
             error_list.append(query_get_row_count)
@@ -143,16 +143,17 @@ class TeradataService(object):
 
         return results
 
-    def get_count_distincts_from_object(self, object : DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns:list=[]) -> dict:
+    def get_count_distincts_from_object(self, object : DatabaseObject, column_intersections: list, where_clause: str="", exclude_columns:list=[],
+        enclose_column_by_double_quotes: bool = False) -> dict:
 
         if self.teradata_connection is None:
             self._connect_to_teradata()
-        
+
         unions=""
         for column in column_intersections:
             if column not in exclude_columns:
                 unions +=f"UNION SELECT CAST('{column}' AS VARCHAR(500)) AS COLUMN_NAME, COUNT(DISTINCT {column}) AS COUNT_DISTINCT FROM {object.schema}.{object.name} {where_clause}"
-        
+
         query_get_count_distincts_from_object=f"{unions[5:]} ORDER BY 2;"
         error_list = []
         dict_count_distincts = []
@@ -167,7 +168,7 @@ class TeradataService(object):
                 }
 
                 dict_count_distincts.append(single_dict)
-        
+
         except Exception as err:
             #raise err
             error_list.append(["ERROR", str(err).split('|||')[0], str(err).split('|||')[1]])
@@ -182,7 +183,8 @@ class TeradataService(object):
 
         return size
 
-    def create_checksums(self, object: DatabaseObject, column_intersections: list, where_clause:str="", exclude_columns:list=[]) -> List[Dict]:
+    def create_checksums(self, object: DatabaseObject, column_intersections: list, where_clause:str="", exclude_columns:list=[],
+            enclose_column_by_double_quotes: bool = False) -> List[Dict]:
 
         if self.teradata_connection is None:
             self._connect_to_teradata()
@@ -212,7 +214,7 @@ class TeradataService(object):
                 elif column_datatype.lower() == 'i1' and 1 == 0:
                     aggregates += f", (SELECT CONCAT ((select trim(count(*)) as val FROM {object.schema}.{object.name} WHERE {column} = 1),'_',(select trim(count(*)) as val from {object.schema}.{object.name} WHERE {column} = 0))) AS aggregateboolean_{column}"
                 #else: Additional Data Types: ++ TD_ANYTYPE, a1 ARRAY, AN ARRAY , bo BINARY LARGE OBJECT, us USER‑DEFINED TYPE (all types),xm XML
-        
+
         query_checksums = f"select {aggregates[1:]} from {object.schema}.{object.name} {where_clause};"
 
         query_countnulls = f"select {count_nulls[1:]} from {object.schema}.{object.name} {where_clause};"
@@ -257,7 +259,19 @@ class TeradataService(object):
         return checksums
 
 
-    def create_pandas_df_from_group_by(self, object: DatabaseObject, column_intersections: list, group_by_columns: list, group_by_aggregation_columns: list, group_by_aggregation_type: str, only_numeric: bool, where_clause: str, exclude_columns: list, numeric_scale: int=None) -> List[Dict]:
+    def create_pandas_df_from_group_by(
+            self,
+            object: DatabaseObject,
+            column_intersections: list,
+            group_by_columns: list,
+            group_by_aggregation_columns: list,
+            group_by_aggregation_type: str,
+            only_numeric: bool,
+            where_clause: str,
+            exclude_columns: list,
+            numeric_scale: int=None,
+            enclose_column_by_double_quotes: bool = False
+        ) -> List[Dict]:
 
         if self.teradata_connection is None:
             self._connect_to_teradata()
@@ -267,7 +281,7 @@ class TeradataService(object):
         else:
             aggregation_columns= [f"{column.upper()}" for column in column_intersections if (column in group_by_aggregation_columns and column not in exclude_columns)]
 
-        dict_colummns_datatype_grouping=self.get_data_types_from_object(object, group_by_columns) 
+        dict_colummns_datatype_grouping=self.get_data_types_from_object(object, group_by_columns)
 
         group_by_query_columns_string = " "
         grouping_columns_final = []
@@ -286,7 +300,7 @@ class TeradataService(object):
 
             group_by_query_columns_string = group_by_query_columns_string[:-1]
 
-            dict_colummns_datatype=self.get_data_types_from_object(object, aggregation_columns)      
+            dict_colummns_datatype=self.get_data_types_from_object(object, aggregation_columns)
 
             aggregates = ""
             aggregates_min = ""
@@ -294,15 +308,15 @@ class TeradataService(object):
             for column in aggregation_columns:
                 column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
                 column_datatype = column_datatype.split('(')[0]
-                
+
                 if column_datatype.lower() == 'i8' or column_datatype.lower() == 'i1' or column_datatype.lower() == 'i' or column_datatype.lower() == 'i2':
-                    
+
                     if not numeric_scale:
                         aggregates += f", sum(cast ({column} as decimal(30,0))) as sum_{column}"
                     else:
                         aggregates += f", CASE WHEN TRIM(TO_CHAR(CAST(ROUND(sum(cast ({column} as decimal(30,0))), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND(sum(cast ({column} as decimal(30,0))), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) ELSE TRIM(TO_CHAR(CAST(ROUND(sum(cast ({column} as decimal(30,0))), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) END as SUM_{column}"
                         aggregates_min += f", CASE WHEN TRIM(TO_CHAR(CAST(ROUND(min({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND(min({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) ELSE TRIM(TO_CHAR(CAST(ROUND(min({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) END as MIN_{column}, CASE WHEN TRIM(TO_CHAR(CAST(ROUND(max({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND(max({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) ELSE TRIM(TO_CHAR(CAST(ROUND(max({column}), {numeric_scale}) as decimal(38,{numeric_scale})), '999999999999999999.{'0'*numeric_scale}')) END as MAX_{column}"
-                
+
                 elif  column_datatype.lower() == 'bf' or column_datatype.lower() == 'bv' or column_datatype.lower() == 'd' or column_datatype.lower() == 'f' or column_datatype.lower() == 'dy' or column_datatype.lower() == 'dh' or column_datatype.lower() == 'dm' or column_datatype.lower() == 'ds' or column_datatype.lower() == 'hr' or column_datatype.lower() == 'hs' or column_datatype.lower() == 'mi' or column_datatype.lower() == 'ms' or column_datatype.lower() == 'mo' or column_datatype.lower() == 'sc' or column_datatype.lower() == 'yr' or column_datatype.lower() == 'ym' or column_datatype.lower() == 'n' or column_datatype.lower() == 'd' :
                     if not numeric_scale:
                         aggregates += f", sum(({column} )) as sum_{column}"
@@ -313,15 +327,15 @@ class TeradataService(object):
 
 
                 elif not only_numeric and ( column_datatype.lower() == 'da' or column_datatype.lower() == 'pd' or column_datatype.lower() == 'pt' or column_datatype.lower() == 'pz' or column_datatype.lower() == 'pm' or column_datatype.lower() == 'at' or column_datatype.lower() == 'ts' or column_datatype.lower() == 'tz' or column_datatype.lower() == 'sz'):
-                    
+
                     aggregates += f", count(distinct {column}) as COUNTDISTINCT_{column}"
                     aggregates_min += f", min({column}) as MIN_{column}, max({column}) as MAX_{column}"
 
                 elif not only_numeric and (column_datatype.lower() == 'cv' or column_datatype.lower() == 'cf' or column_datatype.lower() == 'co'):
-                    
+
                     aggregates += f", count(distinct {column}) as COUNTDISTINCT_{column}"
                     aggregates_min += f", min(TRIM({column})) as MIN_{column}, max(TRIM({column})) as MAX_{column}"
-                    
+
                 elif not only_numeric  and column_datatype.lower() == 'i1' and 1 == 0:
 
                     aggregates += f", (SELECT CONCAT ((select trim(count(*)) as val FROM {object.schema}.{object.name} WHERE {column} = 1),'_',(select trim(count(*)) as val from {object.schema}.{object.name} WHERE {column} = 0))) AS AGGREGATEBOOLEAN_{column}"
@@ -367,7 +381,8 @@ class TeradataService(object):
         return group_by_aggregation_pdf, group_by_query_aggregation_string, group_by_query_columns_string, grouping_columns_final, error_dict
 
 
-    def create_pandas_df(self, object : DatabaseObject, intersection_columns_trgt_src: list, where_clause:str="", exclude_columns:list=[]) -> pd.DataFrame:
+    def create_pandas_df(self, object : DatabaseObject, intersection_columns_trgt_src: list, where_clause:str="", exclude_columns:list=[],
+        enclose_column_by_double_quotes: bool = False) -> pd.DataFrame:
 
         if self.teradata_connection is None:
             self._connect_to_teradata()
@@ -379,8 +394,10 @@ class TeradataService(object):
         src_pdf = self.execute_queries(df_query,True)
 
         return src_pdf
-    
-    def create_pandas_df_from_sample(self, object: DatabaseObject, column_intersections: list,  key_columns: list, where_clause: str="", exclude_columns:list=[], key_filters: dict={}, dedicated_columns: list=[], sample_count :int=10) -> List[Dict]:
+
+    def create_pandas_df_from_sample(self, object: DatabaseObject, column_intersections: list,  key_columns: list, where_clause: str="", exclude_columns:list=[], key_filters: dict={}, dedicated_columns: list=[], sample_count :int=10,
+        numeric_scale: int = None,
+        enclose_column_by_double_quotes: bool = False) -> List[Dict]:
 
         if self.teradata_connection is None:
             self._connect_to_teradata()
@@ -416,12 +433,12 @@ class TeradataService(object):
             for column in dedicated_intersection:
                 column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
                 column_datatype = column_datatype.split('(')[0]
-                
+
                 if column_datatype.lower() == 'i8' or column_datatype.lower() == 'i1' or column_datatype.lower() == 'i' or column_datatype.lower() == 'i2':
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00'))  ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END  as {column}")
                     used_columns.append(column)
                     numeric_columns.append(column)
-                
+
                 elif  column_datatype.lower() == 'bf' or column_datatype.lower() == 'bv' or column_datatype.lower() == 'd' or column_datatype.lower() == 'f' or column_datatype.lower() == 'dy' or column_datatype.lower() == 'dh' or column_datatype.lower() == 'dm' or column_datatype.lower() == 'ds' or column_datatype.lower() == 'hr' or column_datatype.lower() == 'hs' or column_datatype.lower() == 'mi' or column_datatype.lower() == 'ms' or column_datatype.lower() == 'mo' or column_datatype.lower() == 'sc' or column_datatype.lower() == 'yr' or column_datatype.lower() == 'ym' or column_datatype.lower() == 'n' or column_datatype.lower() == 'd' :
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END  as {column}")
                     used_columns.append(column)
@@ -452,22 +469,22 @@ class TeradataService(object):
                     values = list(key_filters.values())
                     if values[0] == []:
                         sample_query = f"SELECT {columns} FROM {object.schema}.{object.name} SAMPLE {sample_count} {where_clause} ORDER BY {keys};"
-                    else:                   
+                    else:
                         where_clause = f'{where_clause} AND (('
                         print(key_filters)
                         for j in range(len(values[0])):
                             for key in key_filters.keys():
                                 if key == 'TECH_ID' or key in numeric_columns:
-                                    where_clause += f" CAST(ROUND({key}, 2) as decimal(38,2)) = {str(key_filters[key][j])} AND" 
-                                else: 
-                                    where_clause += f" {key} = '{str(key_filters[key][j])}' AND" 
+                                    where_clause += f" CAST(ROUND({key}, 2) as decimal(38,2)) = {str(key_filters[key][j])} AND"
+                                else:
+                                    where_clause += f" {key} = '{str(key_filters[key][j])}' AND"
                             where_clause = f" {where_clause[:-3]}) OR ("
                         where_clause = f"{where_clause[:-4]})"
 
                         sample_query = f"SELECT {columns} FROM {object.schema}.{object.name} SAMPLE {sample_count} {where_clause} ORDER BY {keys};"
                 else:
                     sample_query = f"SELECT {columns} FROM {object.schema}.{object.name} SAMPLE {sample_count} {where_clause} ORDER BY {keys};"
-        
+
         elif key_intersection != [] and not is_dedicated:
             column_intersecions_new = []
             used_columns = []
@@ -477,13 +494,13 @@ class TeradataService(object):
             for column in column_intersections:
                 column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
                 column_datatype = column_datatype.split('(')[0]
-                
+
                 if column_datatype.lower() == 'i8' or column_datatype.lower() == 'i1' or column_datatype.lower() == 'i' or column_datatype.lower() == 'i2':
                     #TODO FFR - negativer Fall
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END  as {column}")
                     used_columns.append(column)
                     numeric_columns.append(column)
-                
+
                 elif  column_datatype.lower() == 'bf' or column_datatype.lower() == 'bv' or column_datatype.lower() == 'd' or column_datatype.lower() == 'f' or column_datatype.lower() == 'dy' or column_datatype.lower() == 'dh' or column_datatype.lower() == 'dm' or column_datatype.lower() == 'ds' or column_datatype.lower() == 'hr' or column_datatype.lower() == 'hs' or column_datatype.lower() == 'mi' or column_datatype.lower() == 'ms' or column_datatype.lower() == 'mo' or column_datatype.lower() == 'sc' or column_datatype.lower() == 'yr' or column_datatype.lower() == 'ym' or column_datatype.lower() == 'n' or column_datatype.lower() == 'd' :
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END as {column}")
                     used_columns.append(column)
@@ -504,7 +521,7 @@ class TeradataService(object):
             columns = columns[:-2]
             keys = str(key_intersection)[1:-1].replace("'", "")
 
- 
+
             if key_filters == {}:
                 sample_query = f"SELECT {columns} FROM {object.schema}.{object.name} SAMPLE {sample_count} {where_clause} ORDER BY {keys};"
             else:
@@ -525,7 +542,7 @@ class TeradataService(object):
                     # where_clause += " in " + in_clause
                     if values[0] == []:
                         sample_query = f"SELECT {columns} FROM {object.schema}.{object.name} SAMPLE {sample_count} {where_clause} ORDER BY {keys};"
-                    else:      
+                    else:
                         where_clause = f'{where_clause} AND (('
                         print(key_filters)
                         for j in range(len(values[0])):
@@ -533,7 +550,7 @@ class TeradataService(object):
                                 if key_filters.keys() in numeric_columns:
                                     where_clause += f" {key} = {str(key_filters[key][j])} AND"
                                 else:
-                                    where_clause += f" {key} = '{str(key_filters[key][j])}' AND" 
+                                    where_clause += f" {key} = '{str(key_filters[key][j])}' AND"
                             where_clause += f" {where_clause[:-3]}) OR ("
                         where_clause = f"{where_clause[:-4]})"
 
@@ -552,17 +569,17 @@ class TeradataService(object):
                 print(dict_colummns_datatype)
                 column_datatype=next(x for x in dict_colummns_datatype if x["COLUMN_NAME"] == column)["DATA_TYPE"]
                 column_datatype = column_datatype.split('(')[0]
-                
+
                 if column_datatype.lower() == 'i8' or column_datatype.lower() == 'i1' or column_datatype.lower() == 'i' or column_datatype.lower() == 'i2':
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END  as {column}")
-                    used_columns.append(column) 
+                    used_columns.append(column)
                     numeric_columns.append(column)
-                
+
                 elif  column_datatype.lower() == 'bf' or column_datatype.lower() == 'bv' or column_datatype.lower() == 'd' or column_datatype.lower() == 'f' or column_datatype.lower() == 'dy' or column_datatype.lower() == 'dh' or column_datatype.lower() == 'dm' or column_datatype.lower() == 'ds' or column_datatype.lower() == 'hr' or column_datatype.lower() == 'hs' or column_datatype.lower() == 'mi' or column_datatype.lower() == 'ms' or column_datatype.lower() == 'mo' or column_datatype.lower() == 'sc' or column_datatype.lower() == 'yr' or column_datatype.lower() == 'ym' or column_datatype.lower() == 'n' or column_datatype.lower() == 'd' :
                     column_intersecions_new.append(f"CASE WHEN TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) like '.%' THEN '0' || TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) ELSE TRIM(TO_CHAR(CAST(ROUND({column}, 2) as decimal(38,2)), '999999999999999999999999.00')) END  as {column}")
-                    used_columns.append(column) 
+                    used_columns.append(column)
                     numeric_columns.append(column)
-                    
+
                 elif column_datatype.lower() == 'cv' or column_datatype.lower() == 'cf' or column_datatype.lower() == 'cf':
                     column_intersecions_new.append(f'TRIM({column}) as decimal(38,2)) AS {column}')
                     used_columns.append(column)
@@ -662,4 +679,4 @@ class TeradataService(object):
                 _ = self.teradata_connection.execute(stripped_statement)
 
         except Exception as err:
-            raise Exception(self._get_error_message(err, single_statement)) from err
+            raise Exception(self._get_error_message(err, single_statement)) from err

icsDataValidation/services/system_service.py

@@ -1,6 +1,7 @@
 from icsDataValidation.connection_setups.snowflake_connection_setup import load_snowflake_credentials
 from icsDataValidation.connection_setups.exasol_connection_setup import load_exasol_credentials
 from icsDataValidation.connection_setups.azure_connection_setup import load_azure_credentials
+from icsDataValidation.connection_setups.sqlserver_connection_setup import load_sqlserver_credentials
 from icsDataValidation.connection_setups.teradata_connection_setup import load_teradata_credentials
 from icsDataValidation.connection_setups.oracle_connection_setup import load_oracle_credentials
 from icsDataValidation.connection_setups.databricks_connection_setup import load_databricks_credentials
@@ -8,6 +9,7 @@ from icsDataValidation.services.database_services.snowflake_service import Snowf
 from icsDataValidation.services.database_services.teradata_service import TeradataService
 from icsDataValidation.services.database_services.exasol_service import ExasolService
 from icsDataValidation.services.database_services.azure_service import AzureService
+from icsDataValidation.services.database_services.sqlserver_service import SQLServerService
 from icsDataValidation.services.database_services.oracle_service import OracleService
 from icsDataValidation.services.database_services.databricks_hive_metastore_service import DatabricksHiveMetastoreService
 from icsDataValidation.services.database_services.databricks_unity_catalog_service import DatabricksUnityCatalogService
@@ -33,6 +35,7 @@ class SystemService:
             "SNOWFLAKE": load_snowflake_credentials,
             "EXASOL": load_exasol_credentials,
             "AZURE": load_azure_credentials,
+            "SQLSERVER": load_sqlserver_credentials,
             "TERADATA": load_teradata_credentials,
             "ORACLE": load_oracle_credentials,
             "DATABRICKS_HIVE_METASTORE": load_databricks_credentials,
@@ -52,6 +55,7 @@ class SystemService:
             "SNOWFLAKE": SnowflakeService,
             "EXASOL": ExasolService,
             "AZURE": AzureService,
+            "SQLSERVER": SQLServerService,
             "TERADATA": TeradataService,
             "ORACLE": OracleService,
             "DATABRICKS_HIVE_METASTORE": DatabricksHiveMetastoreService,

icsdatavalidation-1.0.415.dist-info/METADATA

@@ -0,0 +1,298 @@
+Metadata-Version: 2.4
+Name: icsDataValidation
+Version: 1.0.415
+Summary: Add your description here
+Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: azure-storage-blob==12.13.1
+Requires-Dist: boto3==1.26.154
+Requires-Dist: cloe-util-snowflake-connector==1.0.5
+Requires-Dist: databricks-sdk==0.29.0
+Requires-Dist: databricks-sql-connector==3.0.1
+Requires-Dist: numpy==1.26.3
+Requires-Dist: oracledb==2.5.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: pyexasol==0.24.0
+Requires-Dist: pyodbc
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: teradatasql==17.20.0.10
+
+# icsDV - initions Data Validation Tool
+
+## Introduction
+
+The icsDataValidation tool identifies data mismatches between two databases.
+The functionalities are specifically geared to support migration projects.
+It helps to find data issues in tables and views in comparison of a source and a target system.
+
+### What is "generic" about the tool?
+
+The icsDataValidation tool (icsDV) is in particular structered in a way that it is easily expandable.
+The main code is used by all different database options.
+Specifics for each supported database are implemented in a database service per database.
+
+The different database services are very similar.
+They hold the same methods with the same input and output parameters.
+Each method is aligned with the syntax and the settings of the database it is created for.
+Each core implementation includes connections setup, object comparison functionality and the result preparation.
+
+### Supported Databases
+
+The icsDV supports comparisons between the following databases:
+
+- Snowflake
+- Teradata
+- Azure SQL Server
+- Exasol
+- Oracle
+- Databricks with and without Unity Catalog
+
+Comparison results can be written to either Snowflake or Databricks.
+
+### Features
+
+The key features of the tool are:
+
+- Comparison of tables and views between a source and a target system.
+- Pipeline integration in Azure DevOps or GitLab
+- Multiple verification/comparison steps:
+  - Row count comparison
+  - Column names comparison
+  - Aggregation comparison (depending on data type)
+  - "group by" comparison
+  - Pandas DataFrame comparison (with a threshold for the size of the object)
+  - Pandas DataFrame sample comparison (with a random sample of the object)
+- Detailed representation of the comparison result
+  - "high-level" result (for each pipeline/execution)
+  - "object-level" result (for each table/view)
+  - "column-level" result (for each column)
+- Parallelization for performance enhancement of the comparison of a large number of objects
+- Input testsets (white-listing of objects)
+- Object filter (black-listing of objects)
+- Object mappings between the source and the target system
+- Comparison result saved and displayed in multiple instances
+  - saved as JSON files in the repository
+  - export to result tables in the target system (Snowflake or Databricks)
+  - export to Azure Blob Storage or AWS S3 Bucket
+
+### Repository Structure
+
+The repository is structured in the following sections:
+
+- **icsDataValidation**
+  > This is where all code files are stored.
+
+- **icsDataValidation/main.py**
+  > Entry point for python.
+
+- **icsDataValidation/core**
+    > Main code files for the parts independent on the source and target system.
+
+- **icsDataValidation/services/database_services**
+    > Database services for all supported systems can be found here.
+    Each file contains a class that is identically structured in comparison to the other database service classes.
+    Each database service class contains methods to query metadata, create aggregations, and retrieve data for the comparison step.
+
+- **icsDataValidation/connection_setups**
+    > The connection setups are database dependent.
+    They define how the credentials for the database connections are retrieved.
+
+- **examples/comparison_results**
+    > The comparison results are saved here.
+    One JSON file with all results is saved for each execution/pipeline run.
+    Additionally there are live comparison results saved for each compared object as a failsafe.
+
+- **examples**
+    > This folder contains all files defining a specific validation setup.
+      - A file named `migration_config.json` contains configurations about the source system, the target system and the mapping of objects between both. It contains the blacklists and "group by" aggregation settings.
+      - A file named `ics_data_validation_config.json` specifies the source system, the target system and the results system. Most importantly, this includes the names of the results tables and the connection configurations (Server, Port, Secrets) of source and target system.
+      - A file named `manual_execution_params.py` is only relevant for local execution of the code. It contains settings which would otherwise be defined in the pipeline setup, i.e. limits on the size of objects to compare and the numeric precision.
+      - The folder `testsets` contains JSON files specifying whitelists of objects to compare.
+
+    For all the files here, empty `*.template.*` files are available and may serve as a starting point.
+    This repo stores only template files.
+    The actual files used for each setup should not be committed here.
+    They are stored in [a separate repository.](https://dev.azure.com/initions-consulting/icsDataValidation/_git/icsDataValidation%20-%20workflow%20demo).
+
+- **examples/pipeline**
+    > Files defining the pipelines that execute the icsDV are stored here. For example, YML files for Azure DevOps pipelines.
+
+## icsDV - Execution Manual
+
+## icsDV - Input Parameters
+
+There are four types of input parameters:
+
+1. Pipeline Parameters - which are defined as input parameters of a pipeline (Azure DevOps Pipeline or Gitlab Pipeline).
+2. Manual Execution Parameters - defined in the code (testing_tool.py).
+They correspond to the Pipeline Parameters and are used when executing the code directly without a pipeline instead of the Pipeline Parameters.
+3. Global Parameters - directly defined in the TestingToolParams class. They are used in pipeline runs and for manual executions.
+4. Environmental Parameters - Stored either in Azure DevOps in a variable group, in Gitlab, or, for manual executions, in a `*.env` file in a location that can be specified in the `manual_execution_params.py`.
+
+Additionally the parameters can be categorized into 3 groups:
+
+1. Setup Parameters - these are parameters which are usually just set once when setting up the icsDV.
+2. Configuration Parameters - are used to configure the general settings but can be adjusted to the conditions of the workload on the fly.
+3. Execution Parameters - are set individually for each execution of the icsDV, e.g. the selection of objects to be tested.
+
+### Setup Parameters
+
+Stored in `ics_data_validation_config.json`:
+
+| Parameter                             | Description                                                                                                                                                                                                                                                                     | Input Type                                       |
+|---------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------|
+| source_system_selection               | Name of the source system as defined in the database_config.json as a key.                                                                                                                                                                                                      | Pipeline Parameter or Manual Execution Parameter |
+| target_system_selection               | Name of the target system as defined in the database_config.json as a key.                                                                                                                                                                                                      | Pipeline Parameter or Manual Execution Parameter |
+| result_system_selection               | Name of the result system as defined in the database_config.json as a key.                                                                                                                                                                                                      | Pipeline Parameter or Manual Execution Parameter |
+| azure_devops_pipeline                 | Azure DevOps Pipeline support. Set to "True" to push the changes of a run to the GIT repository.                                                                                                                                                                                | Global Parameter - TestingToolParams             |
+| gitlab_pipeline                       | Gitlab Pipeline support. Set to "True" to push the changes of a run to the GIT repository.                                                                                                                                                                                      | Global Parameter - TestingToolParams             |
+| result_database_name                  | Name of the database or catalog the results are written to                                                                                                                                                                                                                      | Global Parameter - TestingToolParams             |
+| result_schema_name                    | Name of the schema the results are written to                                                                                                                                                                                                                                   | Global Parameter - TestingToolParams             |
+| result_table_highlevel_name           | Name of the high-level results table                                                                                                                                                                                                                                            | Global Parameter - TestingToolParams             |
+| result_table_objectlevel_name         | Name of the object-level results table                                                                                                                                                                                                                                          | Global Parameter - TestingToolParams             |
+| result_table_columnlevel_name         | Name of the column-level results table                                                                                                                                                                                                                                          | Global Parameter - TestingToolParams             |
+| result_meta_data_schema_name          | Name of the schema the full results are written to                                                                                                                                                                                                                              | Global Parameter - TestingToolParams             |
+| result_table_name                     | Name of the table the full results are written to                                                                                                                                                                                                                               | Global Parameter - TestingToolParams             |
+| result_live_table_name                | Name of the table the live results are written to                                                                                                                                                                                                                               | Global Parameter - TestingToolParams             |
+| results_folder_name                   | Folder that in which the results are stored in JSON format. Default: `examples/comparison_results/`                                                                                                                                                                             | Global Parameter - TestingToolParams             |
+| remaining_mapping_objects_folder_name | Output folder that holds information about source system objects which are not covered by the mapping and are therefor not included in the comparison. Default: `examples/remaining_mapping_objects/`                                                                           | Global Parameter - TestingToolParams             |
+| testset_folder_name                   | Folder that holds the test set files in JSON format. Default: `examples/testsets/`                                                                                                                                                                                              | Global Parameter - TestingToolParams             |
+| stage_schema                          | Name of the Snowflake Schema where the stage is created to upload the comparison results to Snowflake. Only needed if the `upload_result_to_result_database` functionality is used with Snowflake as target system.                                                             | Global Parameter - TestingToolParams             |
+| stage_name_prefix                     | Prefix of the name of the Snowflake Stage which is used to upload the comparison results to Snowflake. The name is complemented by a run_guid which is a unique uuid for each icsDV execution. Only needed if the `upload_result_to_result_database` functionality is used.     | Global Parameter - TestingToolParams             |
+| container_name                        | Name of the Azure Storage Container to upload the comparison results into the blob storage. Note: Only needed if the `upload_result_to_blob` functionality is used.                                                                                                             | Global Parameter - TestingToolParams             |
+| bucket_name                           | Name of the AWS S3 Bucket to upload the comparison results into the AWS. Note: Only needed if the `upload_result_to_bucket` functionality is used.                                                                                                                              | Global Parameter - TestingToolParams             |
+
+### Configuration Parameters
+
+Stored in `manual_execution_params.py`:
+
+| Parameter                        | Description                                                                                                                                               | Input Type                                        |
+|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|
+| ENV_FILEPATH                     | Absolute path to the `*.env` file containing secrets, passwords and tokens.                                                                               | Pipeline Parameter or Manual Execution Parameters |
+| UPLOAD_RESULT_TO_BLOB            | Set to "True" to upload the comparison results to an Azure Blob Storage. An `azure_storage_connection_string` is needed if set to "True".                 | Pipeline Parameter or Manual Execution Parameters |
+| UPLOAD_RESULT_TO_BUCKET          | Set to "True" to upload the comparison results to an AWS S3 Bucket. An `aws_bucket_access_key` and an `aws_bucket_secret_key` is needed if set to "True". | Pipeline Parameter or Manual Execution Parameter  |
+| UPLOAD_RESULT_TO_RESULT_DATABASE | Set to "True" to upload the comparison results to Snowflake or Databricks. A `result_system_selection` is needed if set to "True".                        | Pipeline Parameter or Manual Execution Parameter  |
+| MAX_OBJECT_SIZE                  | Limits Pandas comparison to objects of a size smaller than `MAX_OBJECT_SIZE` bytes. Data type is String. Default: `str(-1)`, no limit.                    | Pipeline Parameter or Manual Execution Parameter  |
+| MAX_ROW_NUMBER                   | Limits Pandas comparison to objects with less than `MAX_ROW_NUMBER` rows. Data type is String. Default: `str(-1)`, no limit.                              | Pipeline Parameter or Manual Execution Parameter  |
+| EXECUTE_GROUP_BY_COMPARISON      | Set to "True" to execute group-by comparisons. See sec. "Group-By-Aggregation" for details.                                                               | Pipeline Parameter or Manual Execution Parameter  |
+| USE_GROUP_BY_COLUMNS             | Set to "True" to activate group-by columns. See sec. "Group-By-Aggregation" for details.                                                                  | Pipeline Parameter or Manual Execution Parameter  |
+| MIN_GROUP_BY_COUNT_DISTINCT      | Minimum expected number of group-by counts. See sec. "Group-By-Aggregation" for details.                                                                  | Pipeline Parameter or Manual Execution Parameter  |
+| MAX_GROUP_BY_COUNT_DISTINCT      | Maximum expected number of group-by counts. See sec. "Group-By-Aggregation" for details.                                                                  | Pipeline Parameter or Manual Execution Parameter  |
+| MAX_GROUP_BY_SIZE                | Maximum size of the group-by query. See sec. "Group-By-Aggregation" for details.                                                                          | Pipeline Parameter or Manual Execution Parameter  |
+| NUMERIC_SCALE                    | Number of digits to compare. Data type is String. Default: `str(2)`, i.e. deviations below 0.01 are tolerated.                                            | Pipeline Parameter or Manual Execution Parameter  |
+
+### Execution Parameters
+
+Stored in `manual_execution_params.py`:
+
+| Parameter               | Description                                                                                                                                               | Input Type                                       |
+|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------|
+| DATABASE_NAME           | Filters the test set on a specific database/catalog. For no filter set "None" as a Manual Execution Parameter and leave it empty as a Pipeline Parameter. | Pipeline Parameter or Manual Execution Parameter |
+| SCHEMA_NAME             | Filters the test set on a specific schema. For no filter set "None" as a Manual Execution Parameter and leave it empty as a Pipeline Parameter.           | Pipeline Parameter or Manual Execution Parameter |
+| TESTSET_FILE_NAMES      | File names of the test set as defined in the folder testset_folder_name (see Setup Parameters) as JSON files.                                             | Pipeline Parameter or Manual Execution Parameter |
+| OBJECT_TYPE_RESTRICTION | Filters the testset to only tables (`"include_only_tables"`), only views (`"include_only_views"`) or all tables and views (`"include_all"`).              | Pipeline Parameter or Manual Execution Parameter |
+| MAX_NUMBER_OF_THREADS   | Maximum number of threads used. Values larget than the default, `str(1)`, activate parallelization.                                                       | Pipeline Parameter or Manual Execution Parameter |
+
+## icsDV - Configuration
+
+### Blacklists
+
+### Whitelists (Testsets)
+
+### Mapping
+
+### Group-By-Aggregation
+
+The Group-By-Aggregation is a feature to pinpoint the differences in the data.
+It can be activiated by setting the parameter `EXECUTE_GROUP_BY_COMPARISON` to TRUE.
+If activated an additional comparison step is performed.
+Each table is queried with a group-by-statement including aggregations depending on the data type.
+Those aggregations are consequently compared.
+As a result the differences in the data can be narrowed down to certain grouping values.
+
+There are three options to define the column over which the group-by is executed.
+
+1. "group-by-columns-per-table" defined as multiple lists for specific tables. Activated with the `USE_GROUP_BY_COLUMNS` parameter and `GROUP_BY_COLUMNS_PER_TABLE` defined in the `migration_config.json`.
+2. "group-by-columns" from a predifined list for all tables by a validation. Activated with the `USE_GROUP_BY_COLUMNS` parameter and `GROUP_BY_COLUMNS` defined in the `migration_config.json`.
+3. "group-by-columns" evaluated from all existing columns by a validation
+
+The validation consists of a number of tests and can be configured by a number of parameters to either easily find columns to group by over or to only select columns which add a definite value for pinpointing the differences in the data.
+
+The validation tests for the "group-by-columns" are:
+
+  1. Number of distinct values of the column is more than 1.
+  2. Number of distinct values of the column is less than the rowcount of the table.
+  3. Number of distinct values of the column exceeds the `MIN_GROUP_BY_COUNT_DISTINCT` parameter.
+  4. Number of distinct values of the column is below the `MAX_GROUP_BY_COUNT_DISTINCT` parameter.
+  5. The size of the expected result of the group-by-query is below the `MAX_GROUP_BY_SIZE` parameter.
+  (The size is defined by "Number of distinct values" * "Number of columns")
+
+All tests are executed on source and target.
+
+> Note: The group by comparison can be activated by setting the `execute_group_by_comparison` parameter to TRUE.
+The `migration_config.json` has to include the follwing keys when the parameter use_group_by_columns is set to TRUE.
+
+    "GROUP_BY_AGGREGATION":{
+      "GROUP_BY_COLUMNS_PER_TABLE": {},
+      "GROUP_BY_COLUMNS":[]
+    }
+The values of those keys can be empty.
+
+## icsDV - Comparison Results
+
+### JSON Results
+
+- Complete Comparison Result JSONs
+- Live Comparison Result JSONs
+
+### Target System Result Tables
+
+- High-Level Result
+- Object-Level Result
+- Column-Level Result
+
+### Result Export in a File Storage
+
+## icsDV - Setup
+
+### Code setup
+
+- To handle the code, we recommend using VS Code.
+- The code is written in python. The tool is compatible with version 3.11
+- It is recommended to use a project-specific python environment.
+  You can create one with `python -m venv .env` in the root folder of this repo.
+  After creating it, you should activate it (`source .env/bin/activate`), select the python binary `.env/bin/python` therein as your python interpreter in VSC and make sure that python libraries are read from and installed to this environment, i.e. `export PYTHONPATH=$(pwd)/.env/lib/python3.8/site-packages`.
+- In this environment, install the packages listed in the `requirements.txt` and the `requirements-dev.txt`. i.e. run `pip install -r requirements.txt`.
+
+### Setup for manual execution
+
+### Setup as Azure DevOps pipeline
+
+### Setup as GitLab pipeline
+
+## authentication
+
+The following auth methods to snowflake are supported:
+
+- password, provided via PASSWORD_NAME
+- private key with/without encryption, provided via PRIVATE_KEY_NAME with/without PRIVATE_KEY_PASSPHRASE_NAME
+- path to private key file with/without encryption, provided via PRIVATE_KEY_FILE_PATH with/without PRIVATE_KEY_FILE_PASSWORD
+
+## devcontainer
+
+run with uv as follows in devcontainer:
+```bash
+uv run -s  icsDataValidation/main.py
+```
+
+Inside the [devcontainer config](.devcontainer/devcontainer.json) the mounts setting is used to bring a .env from the host system into the devcontainer.
+
+```bash
+"mounts": [
+        "source=/home/Documents/Generic_Testing_Tool/generic_testing_tool_password.env,target=/workspaces/icsDataValidation/examples/generic_testing_tool_password.env,type=bind"
+    ]
+```
+
+To use this feature either create the .env under the source path on your host or adjust this path to another path on the host system. The target path do no need adjustment!