dlt-utils-lib 0.8.0__tar.gz → 0.8.2__tar.gz

{dlt_utils_lib-0.8.0 → dlt_utils_lib-0.8.2}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 0.8.0
+Version: 0.8.2

{dlt_utils_lib-0.8.0 → dlt_utils_lib-0.8.2}/dlt_utils/dlt_transformations.py

@@ -44,3 +44,22 @@ def update_cdc_timestamp(df: DataFrame, time_diff_threshold: int) -> DataFrame:
                 ).otherwise(col('cdc_timestamp'))
             )
     return df
+
+
+def adjust_sequence_operation(df: DataFrame, uniq_columns: list) -> DataFrame:
+    window_spec = Window.partitionBy(uniq_columns).orderBy("cdc_timestamp")
+
+    df_with_lag = df.withColumn("prev_op", lag("Op").over(window_spec)) \
+                    .withColumn("prev_cdc_timestamp", lag("cdc_timestamp").over(window_spec))
+
+    window_spec_all = Window.partitionBy(uniq_columns)
+    df_with_max_u_timestamp = df_with_lag.withColumn(
+        "min_u_cdc_timestamp", spark_min(when(col("Op") == "U", col("cdc_timestamp"))).over(window_spec_all)
+    )
+
+    df_adjusted = df_with_max_u_timestamp.withColumn(
+        "cdc_timestamp",
+        when((col("Op") == "I") & (col("min_u_cdc_timestamp").isNotNull()), col("min_u_cdc_timestamp") -F.expr("interval 1 milliseconds")).otherwise(col("cdc_timestamp"))
+    )
+    df_final = df_adjusted.drop("prev_op", "prev_cdc_timestamp", "min_u_cdc_timestamp")
+    return df_final

{dlt_utils_lib-0.8.0 → dlt_utils_lib-0.8.2}/dlt_utils/main_cdc_utils.py

@@ -1,8 +1,8 @@
 from pyspark.sql.functions import col, expr
-from .dlt_transformations import update_cdc_timestamp, apply_partitions
+from .dlt_transformations import update_cdc_timestamp, apply_partitions, adjust_sequence_operation
 
 # Define the transformation function dynamically
-def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list, time_diff_history_cdc_timestamp: int = 30):
+def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list, keys: list, time_diff_history_cdc_timestamp: int = 30):
     @dlt.table(
         name=f"bronze_{table_name}",
         comment="This is the bronze table.",
@@ -20,6 +20,7 @@ def create_bronze_table_definition(spark, dlt, table_name: str, files_path: str,
             .load(files_path) \
             .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
             .transform(lambda df: update_cdc_timestamp(df, time_diff_history_cdc_timestamp)) \
+            .transform(lambda df: adjust_sequence_operation(df, keys)) \
             .transform(lambda df: apply_partitions(df, partitions))
 
     return transform_cdc_to_bronze

{dlt_utils_lib-0.8.0 → dlt_utils_lib-0.8.2}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 0.8.0
+Version: 0.8.2

{dlt_utils_lib-0.8.0 → dlt_utils_lib-0.8.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='dlt_utils_lib',
-    version='0.8.0',
+    version='0.8.2',
     packages=find_packages(),
     install_requires=[
         'pyspark',