dlt-utils-lib 0.2.2__tar.gz → 0.2.4__tar.gz

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/PKG-INFO

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 0.2.2
+Version: 0.2.4
 Provides-Extra: dev

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/dlt_utils/dlt_transformations.py

@@ -47,3 +47,12 @@ def update_cdc_timestamp(df: DataFrame, time_diff_threshold: int) -> DataFrame:
     return df
 
 
+def add_default_value_for_removed_col(df: DataFrame, default_value_for_removed_col: dict) -> DataFrame:
+    if default_value_for_removed_col:
+        if default_value_for_removed_col['name'] in df.columns:
+            df = df.withColumn(default_value_for_removed_col['name'], when(col(default_value_for_removed_col['name']).isNull(), expr(default_value_for_removed_col['expr'])).otherwise(col(default_value_for_removed_col['name'])))
+        else:
+            df = df.withColumn(default_value_for_removed_col['name'], expr(default_value_for_removed_col['expr']))
+    return df
+
+

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/dlt_utils/main_cdc_utils.py

@@ -1,5 +1,5 @@
 from pyspark.sql.functions import col, expr, coalesce, lit, when
-from .dlt_transformations import update_cdc_timestamp, apply_partitions
+from .dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col
 
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
@@ -53,14 +53,7 @@ def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str,
         temporary=False
     )
     def transform_cdc_to_bronze():
-        df = spark.read.format(file_format).load(files_path)
-
-        if default_value_for_removed_col:
-            if default_value_for_removed_col['name'] in df.columns:
-                df = df.withColumn(default_value_for_removed_col['name'], when(col(default_value_for_removed_col['name']).isNull(), expr(default_value_for_removed_col['expr'])).otherwise(col(default_value_for_removed_col['name'])))
-            else:
-                df = df.withColumn(default_value_for_removed_col['name'], expr(default_value_for_removed_col['expr']))
-        
+        df = spark.read.format(file_format).load(files_path)        
         fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
         schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         return spark \
@@ -73,10 +66,12 @@ def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str,
             .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \
             .transform(lambda df: update_cdc_timestamp(df, time_diff_history_cdc_timestamp)) \
             .transform(lambda df: apply_partitions(df, partitions))
+            .transform(lambda df: add_default_value_for_removed_col(df, default_value_for_removed_col))
 
     return transform_cdc_to_bronze
 
 
+
 # Create silver streaming data
 def silver_streaming_process(dlt, table_name: str, keys: list, partitions: dict, exclude_columns: list):
     dlt.create_streaming_table(

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 0.2.2
+Version: 0.2.4
 Provides-Extra: dev

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='dlt_utils_lib',
-    version='0.2.2',
+    version='0.2.4',
     packages=find_packages(),
     install_requires=[
         'pyspark',

{dlt_utils_lib-0.2.2 → dlt_utils_lib-0.2.4}/tests/test_transformation_module.py

@@ -11,7 +11,7 @@ import warnings
 
 # import for databricks execution whivh not support normal PYTHONPATH :()
 sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "../dlt_utils_lib")))
-from dlt_utils.dlt_transformations import update_cdc_timestamp, apply_partitions
+from dlt_utils.dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col
 
 @pytest.fixture(scope="session")
 def spark() -> SparkSession:
@@ -63,4 +63,34 @@ def test_update_cdc_timestamp(spark):
     assert result[2]['cdc_timestamp'] == datetime(2024, 1, 12)
     
     # The second row should retain its original cdc_timestamp because the difference is less than 5 days
-    assert result[1]['cdc_timestamp'] == datetime(2024, 1, 1)
+    assert result[1]['cdc_timestamp'] == datetime(2024, 1, 1)
+
+
+
+
+def test_add_default_value_for_removed_col(spark):
+    data = [
+        (1, None),
+        (2, 2),
+        (3, None)
+    ]
+    columns = ["id", "raw_data_index"]
+    df = spark.createDataFrame(data, columns)
+
+    default_value_for_removed_col = {
+        'name': 'raw_data_index',
+        'expr': 'cast(0 as int)' 
+    }
+
+    result_df = add_default_value_for_removed_col(df, default_value_for_removed_col)
+
+    expected_data = [
+        (1, 0),  
+        (2, 2),  
+        (3, 0) 
+    ]
+    expected_df = spark.createDataFrame(expected_data, columns)
+    assert result_df.collect() == expected_df.collect()
+
+
+