icsDataValidation 1.0.358__py3-none-any.whl

icsDataValidation/utils/parallelization_util.py

@@ -0,0 +1,52 @@
+import concurrent.futures
+import logging
+import functools
+
+from typing import List
+from icsDataValidation.utils.logger_util import configure_dev_ops_logger
+
+logger = logging.getLogger('Parallelization_Util')
+logger.setLevel(logging.INFO)
+configure_dev_ops_logger(logger)
+
+
+def get_parallelization_groups(objects: list, max_number_of_threads: int) -> List[List]:
+    """
+    Split list of objects into n sublists of preferably same length - with n being the number of threads.
+    The number of threads n is either the input parameter max_number_of_threads or the number of objects.
+    """
+    n_objects = len(objects)
+
+    if max_number_of_threads > n_objects:
+        number_of_threads = n_objects
+    else:
+        number_of_threads = max_number_of_threads
+
+    if not number_of_threads==0:
+        k, m = divmod(n_objects, number_of_threads)
+        object_groups = [
+            objects[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)]
+            for i in range(number_of_threads)
+        ]
+    else:
+        object_groups=[]
+
+    return object_groups, number_of_threads
+
+def execute_func_in_parallel(function, objects: list, max_number_of_threads: int, *function_args) -> list:
+    """
+    Execute a function in parallel. Takes list of objects as input and splits it in object_groups of preferably equal length.
+    The number of threads is equal to the number of object_groups.
+    If the function has arguments they can be passed as additional arguments.
+    """
+    object_groups, number_of_threads = get_parallelization_groups(objects=objects, max_number_of_threads=max_number_of_threads)
+    
+    logger.info(f"Parallelization of function '{function.__name__ }' with number of threads: {number_of_threads}")
+
+    result_groups = []
+    if number_of_threads>0:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=number_of_threads) as pool:
+            partial_function = functools.partial(function, *function_args)
+            for result_group in pool.map(partial_function, object_groups):
+                result_groups.extend(result_group)
+    return result_groups

icsDataValidation/utils/sql_util.py

@@ -0,0 +1,14 @@
+
+
+def parse_filter(filter_list:list)->str:
+    """
+    Retrieve a where-filter in SQL-Syntax from a list of defined conditions.
+    """
+    if filter_list != []:
+        where_clause = "WHERE "
+        for filter_condition in filter_list:
+            where_clause += f"({filter_condition}) AND "
+        where_clause = where_clause[:-4]
+    else:
+        where_clause = ""
+    return where_clause

icsDataValidation-1.0.358.dist-info/METADATA

@@ -0,0 +1,21 @@
+Metadata-Version: 2.2
+Name: icsDataValidation
+Version: 1.0.358
+Summary: Add your description here
+Home-page: https://initions.com/
+Author: initions
+Author-email: ICSMC_EXT_PYPIORG@accenture.com
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: azure-storage-blob==12.13.1
+Requires-Dist: boto3==1.26.154
+Requires-Dist: cloe-util-snowflake-connector==1.0.5
+Requires-Dist: databricks-sdk==0.29.0
+Requires-Dist: databricks-sql-connector==3.0.1
+Requires-Dist: numpy==1.26.3
+Requires-Dist: oracledb==2.5.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: pyexasol==0.24.0
+Requires-Dist: pyodbc
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: teradatasql==17.20.0.10

icsDataValidation-1.0.358.dist-info/RECORD

@@ -0,0 +1,40 @@
+icsDataValidation/configuration.py,sha256=HOFjmC8_e2nvoItndMtJQQA1MR5aCgZGeF1AwY_FvjE,477
+icsDataValidation/main.py,sha256=nmbFM8Epf4-Nhd9ArH31wT7Yx0MSjIHxX93zPke1ArA,12498
+icsDataValidation/connection_setups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/connection_setups/azure_connection_setup.py,sha256=gvTyctG63olZWfFH-qNHcuInMYWcJGWeLpdzciFvZwc,725
+icsDataValidation/connection_setups/databricks_connection_setup.py,sha256=dNEBum-8R-TUW2SCEk3CaNtCr_gLFvn456KBlENpgJU,1220
+icsDataValidation/connection_setups/exasol_connection_setup.py,sha256=RfCUsL6G-NaOW-qNK-3SfHcljbRaKD6fDIHXkNQhClk,590
+icsDataValidation/connection_setups/oracle_connection_setup.py,sha256=D-4ucC1ChE4HYm93ECIEg_yBOrn1NkknxFBgFRGFmWs,978
+icsDataValidation/connection_setups/snowflake_connection_setup.py,sha256=IgEhni4Q0oYGh2QzptpyfEUvUt3cVO28jNSGg11cxyI,1778
+icsDataValidation/connection_setups/teradata_connection_setup.py,sha256=fIpuxz-FTqFK2vSMSuokqU9sdJkaJ4UP5piY_zIbj5k,624
+icsDataValidation/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/core/database_objects.py,sha256=2oaDaVQajSYI_HJjJy1pmc6FsoK_wMfwgu6ZgEcFvow,523
+icsDataValidation/core/object_comparison.py,sha256=OEz5m1pp_PbIWyM5998iB5obFKYdJEqDo9Z0Hpj7o4A,14988
+icsDataValidation/input_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/input_parameters/testing_tool_params.py,sha256=6LkqEaH3vaeCn6d1b3jYIwlXAXIRclxLzcXdsmlkc9M,6731
+icsDataValidation/output_parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/output_parameters/result_params.py,sha256=5Mk9L9zWaxUqcKwLZQ539lVUp0b0s-YUmSA3PBgbqfs,2833
+icsDataValidation/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/services/comparison_service.py,sha256=x8RjZs2bgYDHFueq5ysNADY1cC6rYBf6eDyCsF-_w84,42705
+icsDataValidation/services/initialization_service.py,sha256=AHbJrq_LjMPFoeOJC2pi2ZZ1xkL8njSZn38psc3do60,6687
+icsDataValidation/services/result_service.py,sha256=edD6aejIi5P7qDNHKnN46KrN5tfzwqnw5TB35SvFAWU,28396
+icsDataValidation/services/system_service.py,sha256=GSkSPNG5PlLWchwlYM5H-1FMtuCNwpXcyZZOUB_0stU,3228
+icsDataValidation/services/testset_service.py,sha256=k1wRjI4Ltw9fylek9iW8N6DvnXn13wf6IJ703qQDMEc,15363
+icsDataValidation/services/database_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/services/database_services/azure_service.py,sha256=VTdaT6is_4MMbK9wBJMSJLRFKVRVG-0btjp7dw4izJg,16338
+icsDataValidation/services/database_services/databricks_hive_metastore_service.py,sha256=7UwRazJSPx4A_5wmf9_wt39bHpuyugMQ5HxN6BQAp0w,87581
+icsDataValidation/services/database_services/databricks_unity_catalog_service.py,sha256=INA8rd3KW_jAplNagGa9tEON3dyOufcIAPOOdmc0Mrc,70259
+icsDataValidation/services/database_services/exasol_service.py,sha256=7LYnRScO3DxBmuSN0HmTgsFc2el-Ii3A9jgGsXSJVU8,11074
+icsDataValidation/services/database_services/oracle_service.py,sha256=60unwWlHm520ioFmz0y2K8ApwZrruf9iB0ojjQx0IWc,31523
+icsDataValidation/services/database_services/snowflake_service.py,sha256=EYOZjkjeh0CMGApef-LWoXP4JeJzhAG_qUCqpwOQ9ek,61021
+icsDataValidation/services/database_services/teradata_service.py,sha256=Rf0xzcZGEbooq3r2Rfe2fCahTm2Xw4uznQa8vyWoyqM,40169
+icsDataValidation/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+icsDataValidation/utils/file_util.py,sha256=ZTMB1sTnIIdffg9tEJRCFQQ5SG8Fksc5ie1PM4gHXG4,3432
+icsDataValidation/utils/logger_util.py,sha256=xS48_FFMot_hyQgJY8DUeRTn5jpdvRt5QI6bvlV1jCY,3647
+icsDataValidation/utils/pandas_util.py,sha256=D_g7Xw7BIS2E-1ZhJIvp62K5xuKjIkj-7TxH4HN_8SI,6505
+icsDataValidation/utils/parallelization_util.py,sha256=6P0YcQLmunW_fHR4f5-kdncZbOlxxqKyk6ZAFQQEd2k,2088
+icsDataValidation/utils/sql_util.py,sha256=0c-BInElSsRmXUedfLP_h9Wsiscv9aic7IIc5f15Uzo,396
+icsDataValidation-1.0.358.dist-info/METADATA,sha256=GD_MvcW_v96MzD1qZzY7PJspDVtvGjaCpPGLdW3OyjE,24605
+icsDataValidation-1.0.358.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+icsDataValidation-1.0.358.dist-info/top_level.txt,sha256=BqWUGJb4J7ZybpDMeuGHxEHGHwXXJEIURd9pBybHzTM,18
+icsDataValidation-1.0.358.dist-info/RECORD,,

icsDataValidation-1.0.358.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

icsDataValidation-1.0.358.dist-info/top_level.txt

@@ -0,0 +1 @@
+icsDataValidation