sws-spark-dissemination-helper 0.0.119__tar.gz → 0.0.120__tar.gz

{sws_spark_dissemination_helper-0.0.119 → sws_spark_dissemination_helper-0.0.120}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.119
+Version: 0.0.120
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.119 → sws_spark_dissemination_helper-0.0.120}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.119"
+version = "0.0.120"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.119 → sws_spark_dissemination_helper-0.0.120}/src/sws_spark_dissemination_helper/utils.py

@@ -374,6 +374,26 @@ def map_codes_and_remove_null_duplicates(
                 ),
             ).otherwise(col("note")),
         )
+        # Add mapping message to notes
+        .withColumn(
+            "note",
+            F.when(
+                ~col("is_duplicate")
+                & col("new_dim_code").isNotNull()
+                & (col("new_dim_code") != lit("")),
+                F.array_append(
+                    col("note"),
+                    F.concat(
+                        lit(
+                            f"Dimension {col_name} code was changed from ",
+                            col(col_name),
+                            lit(" to "),
+                            col("new_dim_code"),
+                        )
+                    ),
+                ),
+            ).otherwise(col("note")),
+        )
         .withColumn(
             col_name,
             F.when(