sws-spark-dissemination-helper 0.0.154__tar.gz → 0.0.156__tar.gz

{sws_spark_dissemination_helper-0.0.154 → sws_spark_dissemination_helper-0.0.156}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.154
+Version: 0.0.156
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.154 → sws_spark_dissemination_helper-0.0.156}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.154"
+version = "0.0.156"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.154 → sws_spark_dissemination_helper-0.0.156}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -347,18 +347,25 @@ class SWSBronzeIcebergSparkHelper:
         for dimension_column, df_dimension in zip(
             self.dim_columns_w_time, dfs_dimension_w_validity
         ):
+            logging.info(f"Joining dimension column: {dimension_column}")
+            logging.info(f"df_obs_denorm columns: {df_obs_denorm.columns}")
+            logging.info(
+                f"Is dimension {dimension_column} in the dataframe? {dimension_column in df_obs_denorm.columns}"
+            )
+            df_dimension.show(5)
             df_obs_denorm = (
                 df_obs_denorm.alias("o")
                 .join(
                     F.broadcast(df_dimension.withColumnRenamed("id", "join_id")).alias(
                         "d"
                     ),
-                    col(f"{dimension_column}") == col("d.join_id"),
+                    col(f"{dimension_column}") == col("d.code"),
                 )
-                .drop(f"{dimension_column}", "join_id")
-                .withColumnRenamed("code", dimension_column)
+                .drop("code", "join_id")
             )
-        
+            logging.info(f"After join count: {df_obs_denorm.count()}")
+
+        df_element_uom.show(5)
         df_obs_denorm = (
             df_obs_denorm.alias("d")
             .join(
@@ -368,6 +375,7 @@ class SWSBronzeIcebergSparkHelper:
             )
             .drop("element_code")
         )
+        logging.info(f"After uom count: {df_obs_denorm.count()}")
 
         return df_obs_denorm