sws-spark-dissemination-helper 0.0.180__tar.gz → 0.0.185__tar.gz

{sws_spark_dissemination_helper-0.0.180 → sws_spark_dissemination_helper-0.0.185}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.180
+Version: 0.0.185
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer>=0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: sws-api-client==2.7.3
 Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: tzdata==2025.2
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.180 → sws_spark_dissemination_helper-0.0.185}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.180"
+version = "0.0.185"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.40.0",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer>=0.11.2",
     "six==1.17.0",
-    "sws_api_client==2.3.0",
+    "sws_api_client==2.7.3",
     "typing_extensions>=4.12.2",
     "tzdata==2025.2",
     "urllib3==1.26.20"

{sws_spark_dissemination_helper-0.0.180 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -88,10 +88,13 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    def keep_dim_val_attr_columns(self, df: DataFrame):
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
         cols_to_keep_sws = self.cols_to_keep_sws
-        if "note" in df.columns:
-            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
         if "unit_of_measure_symbol" in df.columns:
             cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
         return df.select(*cols_to_keep_sws)
@@ -156,18 +159,26 @@ class SWSGoldIcebergSparkHelper:
             self.iceberg_tables.SILVER.iceberg_id
         )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
             self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_data(self) -> DataFrame:
-        return self.read_bronze_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame

{sws_spark_dissemination_helper-0.0.180 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -209,7 +209,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("d.diss_note"),
                             F.concat(
-                                col("sy.diss_note"),
+                                col("sy.note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
@@ -225,7 +225,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("new_diss_note"),
                             F.concat(
-                                col("ey.diss_note"),
+                                col("ey.note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),