sws-spark-dissemination-helper 0.0.153__tar.gz → 0.0.155__tar.gz

{sws_spark_dissemination_helper-0.0.153 → sws_spark_dissemination_helper-0.0.155}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.153
+Version: 0.0.155
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.153 → sws_spark_dissemination_helper-0.0.155}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.153"
+version = "0.0.155"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.153 → sws_spark_dissemination_helper-0.0.155}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -339,6 +339,42 @@ class SWSBronzeIcebergSparkHelper:
 
         df_obs_denorm = self.spark.sql(final_query)
 
+        df_element_uom = self._prepare_element_uom()
+
+        dfs_dimension_w_validity = self._convert_dim_start_end_date_to_data()
+
+        # Join all the dimension codelists
+        for dimension_column, df_dimension in zip(
+            self.dim_columns_w_time, dfs_dimension_w_validity
+        ):
+            logging.info(f"Joining dimension column: {dimension_column}")
+            logging.info(f"df_obs_denorm columns: {df_obs_denorm.columns}")
+            logging.info(
+                f"Is dimension {dimension_column} in the dataframe? {dimension_column in df_obs_denorm.columns}"
+            )
+            df_dimension.show(5)
+            df_obs_denorm = (
+                df_obs_denorm.alias("o")
+                .join(
+                    F.broadcast(df_dimension.withColumnRenamed("id", "join_id")).alias(
+                        "d"
+                    ),
+                    col(f"{dimension_column}") == col("d.code"),
+                )
+                .drop("code", "join_id")
+            )
+
+        df_element_uom.show(5)
+        df_obs_denorm = (
+            df_obs_denorm.alias("d")
+            .join(
+                F.broadcast(df_element_uom).alias("e"),
+                col(f"d.{self.element_column}") == col("e.element_code"),
+                "left",
+            )
+            .drop("element_code")
+        )
+
         return df_obs_denorm
 
     def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame: