dlt-utils-lib 0.1__0-py3-none-any.whl

dlt_utils/dlt_transformation.py

@@ -0,0 +1,23 @@
+def apply_partitions(df: DataFrame, partitions: dict):
+    # apply partitioning to the dataframe
+    if partitions:
+        for col_name, expression in partitions.items():
+            if expression.replace(" ", "") != '':
+                df = df.withColumn(col_name, expr(expression))
+    return df
+    
+def update_cdc_timestamp(df: DataFrame) -> DataFrame:
+    # if cdc_timestamp is null or time difference is greater than threshold, set it to the max timestamp in the row
+    time_diff_threshold = 5 # 5 days
+    timestamp_cols = [col.name for col in df.schema.fields if isinstance(col.dataType, (TimestampType, DateType)) and col.name != 'cdc_timestamp']
+    if timestamp_cols:
+        max_timestamp_per_row = greatest(*[col(col_name) for col_name in timestamp_cols])
+        df = df.withColumn(
+            'cdc_timestamp',
+            when(
+                col('cdc_timestamp').isNull() | 
+                (spark_abs(datediff(max_timestamp_per_row, col('cdc_timestamp'))) > time_diff_threshold), 
+                max_timestamp_per_row
+            ).otherwise(col('cdc_timestamp'))
+        )
+    return df

dlt_utils/dlt_transformations.py

@@ -0,0 +1,90 @@
+from pyspark.sql import DataFrame
+from pyspark.sql.types import TimestampType, DateType
+from pyspark.sql.functions import col, expr, greatest, when, datediff, lag, min as spark_min, expr, count, coalesce
+from pyspark.sql.window import Window
+
+
+def apply_partitions(df: DataFrame, partitions: dict):
+    # apply partitioning to the dataframe
+    if partitions:
+        for col_name, expression in partitions.items():
+            if expression.replace(" ", "") != '':
+                df = df.withColumn(col_name, expr(expression))
+    return df
+    
+def update_cdc_timestamp(df: DataFrame, time_diff_threshold: int) -> DataFrame:
+    # if cdc_timestamp is null or time difference is greater than threshold, set it to the max timestamp in the row
+    timestamp_cols = [col.name for col in df.schema.fields if isinstance(col.dataType, (TimestampType, DateType)) and col.name != 'cdc_timestamp']
+
+    if timestamp_cols:
+        max_timestamp_per_row = None
+
+        if len(timestamp_cols) > 1:
+            max_timestamp_per_row = greatest(*[col(col_name) for col_name in timestamp_cols])
+            # temp fix for market table if possible need take last_update_date becouse of market table contnin close time
+            if 'last_update_date' in timestamp_cols:
+                max_timestamp_per_row = when(
+                                                col('last_update_date').isNotNull(), 
+                                                col('last_update_date')
+                                            ).otherwise(max_timestamp_per_row)
+            # temp fix for artemis table if possible need take last_update_date becouse of market table contnin close time
+            if 'version' in timestamp_cols:
+                max_timestamp_per_row = when(
+                                                col('version').isNotNull(), 
+                                                col('version')
+                                            ).otherwise(max_timestamp_per_row)        
+        else:
+            max_timestamp_per_row = col(timestamp_cols[0])
+            
+        df = df.withColumn(
+            'cdc_timestamp',
+                when(
+                    (col('cdc_timestamp').isNull()) | 
+                    (datediff(col('cdc_timestamp'), max_timestamp_per_row) > time_diff_threshold), 
+                    max_timestamp_per_row
+                ).otherwise(col('cdc_timestamp'))
+            )
+    return df
+
+def compare_cdc_timestamp_and_commit_timestamp(df, uniq_columns):
+    
+    if 'commit_timestamp' in df.columns:
+        df = df.withColumn('commit_timestamp', coalesce(col('commit_timestamp').cast('timestamp'), col('cdc_timestamp').cast('timestamp')))
+    else:
+        df = df.withColumn('commit_timestamp', col('cdc_timestamp').cast('timestamp'))
+
+    columns = uniq_columns + ['cdc_timestamp']
+    df = df.withWatermark('cdc_timestamp', '5 minutes')  
+
+    grouped_df = df.groupBy(columns).agg(
+        count("*").alias("record_count"),
+    )
+    df_with_conditions = df.join(grouped_df, on=columns, how="left")
+    
+    df_adjusted = df_with_conditions.withColumn(
+        "cdc_timestamp",
+        when((col("record_count") > 1) & (col("commit_timestamp").isNotNull()), col("commit_timestamp"))
+         .otherwise(col("cdc_timestamp"))
+    )
+    df_adjusted = df_adjusted.drop("record_count")
+    return df_adjusted
+
+
+
+def adjust_sequence_operation(df: DataFrame, uniq_columns: list) -> DataFrame:
+    window_spec = Window.partitionBy(uniq_columns).orderBy("cdc_timestamp")
+
+    df_with_lag = df.withColumn("prev_op", lag("Op").over(window_spec)) \
+                    .withColumn("prev_cdc_timestamp", lag("cdc_timestamp").over(window_spec))
+
+    window_spec_all = Window.partitionBy(uniq_columns)
+    df_with_max_u_timestamp = df_with_lag.withColumn(
+        "min_u_cdc_timestamp", spark_min(when(col("Op") == "U", col("cdc_timestamp"))).over(window_spec_all)
+    )
+
+    df_adjusted = df_with_max_u_timestamp.withColumn(
+        "cdc_timestamp",
+        when((col("Op") == "I") & (col("min_u_cdc_timestamp").isNotNull()), col("min_u_cdc_timestamp") -expr("interval 1 milliseconds")).otherwise(col("cdc_timestamp"))
+    )
+    df_final = df_adjusted.drop("prev_op", "prev_cdc_timestamp", "min_u_cdc_timestamp")
+    return df_final

dlt_utils/dlt_utils.py

@@ -0,0 +1,44 @@
+from pyspark.sql.functions import col, expr
+import dlt
+
+def create_bronze_table_definition(table_name, files_path, file_format, schema_exclude_columns):
+    @dlt.table(
+        name=f"bronze_{table_name}",
+        comment="This is the bronze table.",
+        temporary=False
+    )
+    def transform_cdc_to_bronze():
+        df = spark.read.format(file_format).load(files_path)
+        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
+        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
+        return spark \
+            .readStream \
+            .format('cloudFiles') \
+            .option("cloudFiles.format", file_format) \
+            .option("cloudFiles.schemaHints", schema_string) \
+            .load(files_path) \
+            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp'))
+
+    return transform_cdc_to_bronze
+
+def silver_streaming_process(table_name, keys, partition_col, exclude_columns):
+    dlt.create_streaming_table(
+        name=table_name,
+        table_properties={
+            "delta.autoOptimize.optimizeWrite": "true",
+            "delta.autoOptimize.autoCompact": "true"
+        },
+        comment="This is the silver table with source in",
+        partition_cols=partition_col if partition_col else None
+    )
+
+    dlt.apply_changes(
+        target=table_name,
+        source=f"bronze_{table_name}",
+        keys=keys,
+        sequence_by=col("cdc_timestamp"),
+        apply_as_deletes=expr("Op = 'D'"),
+        apply_as_truncates=expr("Op = 'T'"),
+        except_column_list=["Op", "_rescued_data"] + exclude_columns,
+        stored_as_scd_type=1
+    )

dlt_utils/main_cdc_utils.py

@@ -0,0 +1,85 @@
+from pyspark.sql.functions import col, expr, coalesce, lit
+from .dlt_transformations import update_cdc_timestamp, apply_partitions
+
+# Base code pipeline: streaming process
+def base_cdc_replication_process(dlt, spark,  table_name: str, files_path: str, bucket_name: str, cdc_data_list_maps: list, bronze_directory: str,time_diff_history_cdc_timestamp: int = 30):
+
+    # parse list with tables for replication
+    for table in cdc_data_list_maps:
+        source_table_name = table['source_table_name']
+        source_schema = table['source_schema']
+        file_format = table['format']
+        keys = table['keys']
+        exclude_columns = table['exclude_columns']
+        partition_col = table['partition_col']
+
+        s3_files_path = f"{source_schema}/{source_table_name}/"
+        bronze_table_path = f"{bucket_name}/{bronze_directory}/{s3_files_path}"
+
+
+        # Execute pipeline
+        create_bronze_table_definition(spark=spark,
+                                       dlt=dlt,
+                                       table_name=source_table_name,
+                                       files_path=bronze_table_path, 
+                                       file_format=file_format,
+                                       partitions=partition_col,
+                                       schema_exclude_columns=exclude_columns,
+                                       keys=keys,
+                                       time_diff_history_cdc_timestamp=time_diff_history_cdc_timestamp
+                                        )
+        
+        silver_streaming_process(dlt=dlt,
+                                table_name=source_table_name,
+                                keys=keys,
+                                partitions=partition_col,
+                                exclude_columns=exclude_columns)
+        
+
+# Define the transformation function dynamically
+def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list, keys: list, time_diff_history_cdc_timestamp: int):
+    @dlt.table(
+        name=f"bronze_{table_name}",
+        comment="This is the bronze table.",
+        temporary=False
+    )
+    def transform_cdc_to_bronze():
+        df = spark.read.format(file_format).load(files_path)
+        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
+        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
+        return spark \
+            .readStream \
+            .format('cloudFiles') \
+            .option("cloudFiles.format", file_format) \
+            .option("cloudFiles.schemaHints", schema_string) \
+            .load(files_path) \
+            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
+            .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \
+            .transform(lambda df: update_cdc_timestamp(df, time_diff_history_cdc_timestamp)) \
+            .transform(lambda df: apply_partitions(df, partitions))
+
+    return transform_cdc_to_bronze
+
+
+# Create silver streaming data
+def silver_streaming_process(dlt, table_name: str, keys: list, partitions: dict, exclude_columns: list):
+    dlt.create_streaming_table(
+        name=table_name,
+        table_properties={
+            "delta.autoOptimize.optimizeWrite": "true",
+            "delta.autoOptimize.autoCompact": "true"
+        },
+        comment = "This is the silver table with source in",
+        partition_cols=partitions if partitions else None
+    )
+
+    dlt.apply_changes(
+        target=table_name,
+        source=f"bronze_{table_name}",
+        keys=keys,
+        sequence_by=col("ar_h_change_seq"),
+        apply_as_deletes=expr("Op = 'D'"),
+        apply_as_truncates=expr("Op = 'T'"),
+        except_column_list=["Op", "_rescued_data"] + exclude_columns,
+        stored_as_scd_type=1
+    )

dlt_utils/main_dlt_utils.py

@@ -0,0 +1,59 @@
+import dlt
+from pyspark.sql.functions import col, expr, greatest, when, datediff, abs as spark_abs
+from dlt_transformations import update_cdc_timestamp, apply_partitions
+from pyspark.sql.types import TimestampType, DateType
+from pyspark.sql import DataFrame
+from pyspark.sql import SparkSession
+
+# Assuming SparkSession is already created in your Databricks notebook
+spark = SparkSession.builder.appName("DLT Pipeline").getOrCreate()
+spark.conf.set("pipelines.tableManagedByMultiplePipelinesCheck.enabled", "false")
+
+
+# Define the transformation function dynamically
+def create_bronze_table_definition(table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list):
+    @dlt.table(
+        name=f"bronze_{table_name}",
+        comment="This is the bronze table.",
+        temporary=False
+    )
+    def transform_cdc_to_bronze():
+        df = spark.read.format(file_format).load(files_path)
+        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
+        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
+        return spark \
+            .readStream \
+            .format('cloudFiles') \
+            .option("cloudFiles.format", file_format) \
+            .option("cloudFiles.schemaHints", schema_string) \
+            .load(files_path) \
+            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
+            .transform(update_cdc_timestamp) \
+            .transform(lambda df: apply_partitions(df, partitions))
+
+    return transform_cdc_to_bronze
+
+
+
+# Create silver streaming data
+def silver_streaming_process(table_name: str, keys: list, partitions: dict, exclude_columns: list):
+    dlt.create_streaming_table(
+        name=table_name,
+        table_properties={
+            "delta.autoOptimize.optimizeWrite": "true",
+            "delta.autoOptimize.autoCompact": "true"
+        },
+        comment = "This is the silver table with source in",
+        partition_cols=partitions if partitions else None
+    )
+
+    dlt.apply_changes(
+        target=table_name,
+        source=f"bronze_{table_name}",
+        keys=keys,
+        sequence_by=col("cdc_timestamp"),
+        apply_as_deletes=expr("Op = 'D'"),
+        apply_as_truncates=expr("Op = 'T'"),
+        except_column_list=["Op", "_rescued_data"] + exclude_columns,
+        stored_as_scd_type=1
+    )

dlt_utils_lib-0.1.0.dist-info/METADATA

@@ -0,0 +1,6 @@
+Metadata-Version: 2.1
+Name: dlt-utils-lib
+Version: 0.1.0
+Requires-Dist: pyspark
+Requires-Dist: delta-spark
+

dlt_utils_lib-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dlt_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dlt_utils/dlt_transformation.py,sha256=PPvNi4pVtP5fM69Zpzm1VvagA0e93B7augVzEB_WeGU,1109
+dlt_utils/dlt_transformations.py,sha256=R1bCH3JtidIK0embVqbH-gBdkkpiYG6F2WVdUJxkT5A,4258
+dlt_utils/dlt_utils.py,sha256=KzZjLgWtACns3kQJI9O-1F030xCvgJPDcT3T6Il0NZg,1691
+dlt_utils/main_cdc_utils.py,sha256=PI9oFiYOP0dczRqX5iguP8FZqdR7k6VGeS9KjmLEeWI,3907
+dlt_utils/main_dlt_utils.py,sha256=goPB2SsoHx6IeX7BYSn_eGZmyxA4emN-tT6Px-LK1Ao,2408
+dlt_utils_lib-0.1.0.dist-info/METADATA,sha256=14Gb20-s7sH2rsjyamzWSfFYyY9lPB2utVXAd_-H8Rk,108
+dlt_utils_lib-0.1.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+dlt_utils_lib-0.1.0.dist-info/tmpz2kz6zic,sha256=ovntz9H59jTF7MwMFD9bqx2-k_fa759JGnM8ENGfr7s,105
+dlt_utils_lib-0.1.0.dist-info/top_level.txt,sha256=KQIiBkukfDFtKV6NmbNK651_k5Oj0jLk5dpU0ZO1xuQ,10
+dlt_utils_lib-0.1.0.dist-info/RECORD,,

dlt_utils_lib-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.38.4)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dlt_utils_lib-0.1.0.dist-info/tmpz2kz6zic

@@ -0,0 +1,5 @@
+Metadata-Version: 2.1
+Name: dlt_utils_lib
+Version: 0.4
+Requires-Dist: pyspark
+Requires-Dist: delta-spark

dlt_utils_lib-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dlt_utils