dlt-utils-lib 0.2.4__tar.gz → 0.2.6__tar.gz

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/PKG-INFO

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 0.2.4
+Version: 0.2.6
 Provides-Extra: dev

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/dlt_utils/dlt_transformations.py

@@ -56,3 +56,12 @@ def add_default_value_for_removed_col(df: DataFrame, default_value_for_removed_c
     return df
 
 
+from pyspark.sql import DataFrame
+
+def rename_columns(df: DataFrame) -> DataFrame:
+    new_columns = [col.replace('.', '_') for col in df.columns]
+    return df.toDF(*new_columns)
+
+
+
+

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/dlt_utils/main_cdc_utils.py

@@ -1,5 +1,5 @@
 from pyspark.sql.functions import col, expr, coalesce, lit, when
-from .dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col
+from .dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col, rename_columns
 
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
@@ -65,8 +65,9 @@ def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str,
             .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
             .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \
             .transform(lambda df: update_cdc_timestamp(df, time_diff_history_cdc_timestamp)) \
-            .transform(lambda df: apply_partitions(df, partitions))
-            .transform(lambda df: add_default_value_for_removed_col(df, default_value_for_removed_col))
+            .transform(lambda df: apply_partitions(df, partitions)) \
+            .transform(lambda df: add_default_value_for_removed_col(df, default_value_for_removed_col)) \
+            .transform(rename_columns)
 
     return transform_cdc_to_bronze
 

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 0.2.4
+Version: 0.2.6
 Provides-Extra: dev

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='dlt_utils_lib',
-    version='0.2.4',
+    version='0.2.6',
     packages=find_packages(),
     install_requires=[
         'pyspark',

{dlt_utils_lib-0.2.4 → dlt_utils_lib-0.2.6}/tests/test_transformation_module.py

@@ -11,7 +11,7 @@ import warnings
 
 # import for databricks execution whivh not support normal PYTHONPATH :()
 sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "../dlt_utils_lib")))
-from dlt_utils.dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col
+from dlt_utils.dlt_transformations import update_cdc_timestamp, apply_partitions, add_default_value_for_removed_col, rename_columns
 
 @pytest.fixture(scope="session")
 def spark() -> SparkSession:
@@ -94,3 +94,11 @@ def test_add_default_value_for_removed_col(spark):
 
 
 
+def test_rename_columns(spark):
+    data = [("Alice", 1), ("Bob", 2)]
+    columns = ["name.first", "age"]
+    df = spark.createDataFrame(data, columns)    
+    df = df.transform(rename_columns)
+    expected_columns = ["name_first", "age"]
+    assert df.columns == expected_columns
+