sws-spark-dissemination-helper 0.0.109__tar.gz → 0.0.111__tar.gz

{sws_spark_dissemination_helper-0.0.109 → sws_spark_dissemination_helper-0.0.111}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.109
+Version: 0.0.111
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.109 → sws_spark_dissemination_helper-0.0.111}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.109"
+version = "0.0.111"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.109 → sws_spark_dissemination_helper-0.0.111}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -172,6 +172,10 @@ class SWSSilverIcebergSparkHelper:
             )
 
             original_col_order = df.columns
+            cols_to_select = df.columns
+            cols_to_select.remove("note")
+            cols_to_select.remove(f"{col_name}_start_date")
+            cols_to_select.remove(f"{col_name}_end_date")
 
             df = (
                 df.alias("d")
@@ -188,7 +192,7 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
                 .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
                 .withColumn(
-                    "note",
+                    "new_note",
                     F.when(
                         col("valid_new_start_year"),
                         F.array_append(
@@ -201,14 +205,14 @@ class SWSSilverIcebergSparkHelper:
                                 col("sy.new_code"),
                             ),
                         ),
-                    ).otherwise(col("note")),
+                    ).otherwise(col("d.note")),
                 )
                 .withColumn(
-                    "note",
+                    "new_note",
                     F.when(
                         col("valid_new_end_year"),
                         F.array_append(
-                            col("note"),
+                            col("new_note"),
                             F.concat(
                                 col("ey.note"),
                                 lit(" from "),
@@ -217,22 +221,25 @@ class SWSSilverIcebergSparkHelper:
                                 col("ey.new_code"),
                             ),
                         ),
-                    ).otherwise(col("note")),
+                    ).otherwise(col("new_note")),
                 )
                 .withColumn(
-                    f"{col_name}_start_date",
+                    f"new_{col_name}_start_date",
                     F.when(
                         col("valid_new_start_year"), F.to_date(col("sy.new_code"))
                     ).otherwise(col(f"d.{col_name}_start_date")),
                 )
                 .withColumn(
-                    f"{col_name}_end_date",
+                    f"new_{col_name}_end_date",
                     F.when(
                         col("valid_new_end_year"), F.to_date(col("ey.new_code"))
                     ).otherwise(col(f"d.{col_name}_end_date")),
                 )
                 .selectExpr(
-                    "d.*", "note", f"{col_name}_start_date", f"{col_name}_end_date"
+                    *cols_to_select,
+                    col("new_note").alias("note"),
+                    col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),
+                    col(f"new_{col_name}_end_date").alias(f"{col_name}_end_date"),
                 )
                 .select(*original_col_order)
             )