dlt-utils-lib 0.1__py3-none-any.whl

dlt_utils/dlt_transformation.py

@@ -0,0 +1,23 @@
+def apply_partitions(df: DataFrame, partitions: dict):
+    # apply partitioning to the dataframe
+    if partitions:
+        for col_name, expression in partitions.items():
+            if expression.replace(" ", "") != '':
+                df = df.withColumn(col_name, expr(expression))
+    return df
+    
+def update_cdc_timestamp(df: DataFrame) -> DataFrame:
+    # if cdc_timestamp is null or time difference is greater than threshold, set it to the max timestamp in the row
+    time_diff_threshold = 5 # 5 days
+    timestamp_cols = [col.name for col in df.schema.fields if isinstance(col.dataType, (TimestampType, DateType)) and col.name != 'cdc_timestamp']
+    if timestamp_cols:
+        max_timestamp_per_row = greatest(*[col(col_name) for col_name in timestamp_cols])
+        df = df.withColumn(
+            'cdc_timestamp',
+            when(
+                col('cdc_timestamp').isNull() | 
+                (spark_abs(datediff(max_timestamp_per_row, col('cdc_timestamp'))) > time_diff_threshold), 
+                max_timestamp_per_row
+            ).otherwise(col('cdc_timestamp'))
+        )
+    return df

dlt_utils/dlt_utils.py

@@ -0,0 +1,44 @@
+from pyspark.sql.functions import col, expr
+import dlt
+
+def create_bronze_table_definition(table_name, files_path, file_format, schema_exclude_columns):
+    @dlt.table(
+        name=f"bronze_{table_name}",
+        comment="This is the bronze table.",
+        temporary=False
+    )
+    def transform_cdc_to_bronze():
+        df = spark.read.format(file_format).load(files_path)
+        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
+        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
+        return spark \
+            .readStream \
+            .format('cloudFiles') \
+            .option("cloudFiles.format", file_format) \
+            .option("cloudFiles.schemaHints", schema_string) \
+            .load(files_path) \
+            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp'))
+
+    return transform_cdc_to_bronze
+
+def silver_streaming_process(table_name, keys, partition_col, exclude_columns):
+    dlt.create_streaming_table(
+        name=table_name,
+        table_properties={
+            "delta.autoOptimize.optimizeWrite": "true",
+            "delta.autoOptimize.autoCompact": "true"
+        },
+        comment="This is the silver table with source in",
+        partition_cols=partition_col if partition_col else None
+    )
+
+    dlt.apply_changes(
+        target=table_name,
+        source=f"bronze_{table_name}",
+        keys=keys,
+        sequence_by=col("cdc_timestamp"),
+        apply_as_deletes=expr("Op = 'D'"),
+        apply_as_truncates=expr("Op = 'T'"),
+        except_column_list=["Op", "_rescued_data"] + exclude_columns,
+        stored_as_scd_type=1
+    )

dlt_utils/main_dlt_utils.py

@@ -0,0 +1,59 @@
+import dlt
+from pyspark.sql.functions import col, expr, greatest, when, datediff, abs as spark_abs
+from dlt_transformations import update_cdc_timestamp, apply_partitions
+from pyspark.sql.types import TimestampType, DateType
+from pyspark.sql import DataFrame
+from pyspark.sql import SparkSession
+
+# Assuming SparkSession is already created in your Databricks notebook
+spark = SparkSession.builder.appName("DLT Pipeline").getOrCreate()
+spark.conf.set("pipelines.tableManagedByMultiplePipelinesCheck.enabled", "false")
+
+
+# Define the transformation function dynamically
+def create_bronze_table_definition(table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list):
+    @dlt.table(
+        name=f"bronze_{table_name}",
+        comment="This is the bronze table.",
+        temporary=False
+    )
+    def transform_cdc_to_bronze():
+        df = spark.read.format(file_format).load(files_path)
+        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
+        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
+        return spark \
+            .readStream \
+            .format('cloudFiles') \
+            .option("cloudFiles.format", file_format) \
+            .option("cloudFiles.schemaHints", schema_string) \
+            .load(files_path) \
+            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
+            .transform(update_cdc_timestamp) \
+            .transform(lambda df: apply_partitions(df, partitions))
+
+    return transform_cdc_to_bronze
+
+
+
+# Create silver streaming data
+def silver_streaming_process(table_name: str, keys: list, partitions: dict, exclude_columns: list):
+    dlt.create_streaming_table(
+        name=table_name,
+        table_properties={
+            "delta.autoOptimize.optimizeWrite": "true",
+            "delta.autoOptimize.autoCompact": "true"
+        },
+        comment = "This is the silver table with source in",
+        partition_cols=partitions if partitions else None
+    )
+
+    dlt.apply_changes(
+        target=table_name,
+        source=f"bronze_{table_name}",
+        keys=keys,
+        sequence_by=col("cdc_timestamp"),
+        apply_as_deletes=expr("Op = 'D'"),
+        apply_as_truncates=expr("Op = 'T'"),
+        except_column_list=["Op", "_rescued_data"] + exclude_columns,
+        stored_as_scd_type=1
+    )

dlt_utils_lib-0.1.dist-info/METADATA

@@ -0,0 +1,6 @@
+Metadata-Version: 2.1
+Name: dlt_utils_lib
+Version: 0.1
+Requires-Dist: pyspark
+Requires-Dist: delta-spark
+

dlt_utils_lib-0.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+dlt_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dlt_utils/dlt_transformation.py,sha256=PPvNi4pVtP5fM69Zpzm1VvagA0e93B7augVzEB_WeGU,1109
+dlt_utils/dlt_utils.py,sha256=KzZjLgWtACns3kQJI9O-1F030xCvgJPDcT3T6Il0NZg,1691
+dlt_utils/main_dlt_utils.py,sha256=goPB2SsoHx6IeX7BYSn_eGZmyxA4emN-tT6Px-LK1Ao,2408
+dlt_utils_lib-0.1.dist-info/METADATA,sha256=AXnvlFI4UetjhIEmvUbAiUb3KrhOXRYkBOepSDrdaQg,106
+dlt_utils_lib-0.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+dlt_utils_lib-0.1.dist-info/top_level.txt,sha256=KQIiBkukfDFtKV6NmbNK651_k5Oj0jLk5dpU0ZO1xuQ,10
+dlt_utils_lib-0.1.dist-info/RECORD,,

dlt_utils_lib-0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dlt_utils_lib-0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+dlt_utils