sws-spark-dissemination-helper 0.0.177__tar.gz → 0.0.179__tar.gz

{sws_spark_dissemination_helper-0.0.177 → sws_spark_dissemination_helper-0.0.179}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.177
+Version: 0.0.179
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.177 → sws_spark_dissemination_helper-0.0.179}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.177"
+version = "0.0.179"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.40.0",

{sws_spark_dissemination_helper-0.0.177 → sws_spark_dissemination_helper-0.0.179}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -96,19 +96,19 @@ class SWSGoldIcebergSparkHelper:
             cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
         return df.select(*cols_to_keep_sws)
 
-    def add_display_decimals(self, df: DataFrame):
+    def round_to_display_decimals(self, df: DataFrame):
         col1_name, col2_name = (
             self.display_decimals.select("column_1_name", "column_2_name")
             .distinct()
             .collect()[0]
         )
-        if col1_name.lower() not in [column.lower() for column in df.columns()]:
+        if col1_name.lower() not in [column.lower() for column in df.columns]:
             raise ValueError(
-                f"{col1_name} is not part of the columns available for this dataset ({df.columns()})"
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns})"
             )
-        if col2_name.lower() not in [column.lower() for column in df.columns()]:
+        if col2_name.lower() not in [column.lower() for column in df.columns]:
             raise ValueError(
-                f"{col2_name} is not part of the columns available for this dataset ({df.columns()})"
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns})"
             )
 
         df = (
@@ -130,14 +130,20 @@ class SWSGoldIcebergSparkHelper:
         )
 
         df = df.withColumn(
-            "display_decimals", F.coalesce(col("display_decimals"), lit("0"))
+            "display_decimals",
+            F.coalesce(col("display_decimals"), lit("0")).cast("INT"),
         ).withColumn(
             "value",
             F.round(
-                col("value").cast("FLOAT"), col("display_decimals").cast("FLOAT")
-            ).cast("STRING"),
+                F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
+            )
+            / F.pow(10, F.col("dec")).cast("STRING"),
         )
 
+        # F.round(
+        #     col("value").cast("FLOAT"), col("display_decimals").cast("INT")
+        # ).cast("STRING"),
+
         return df
 
     def read_bronze_data(self) -> DataFrame: