sws-spark-dissemination-helper 0.0.156__tar.gz → 0.0.157__tar.gz

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.157}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.156
+Version: 0.0.157
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.157}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.156"
+version = "0.0.157"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.157}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -347,12 +347,11 @@ class SWSBronzeIcebergSparkHelper:
         for dimension_column, df_dimension in zip(
             self.dim_columns_w_time, dfs_dimension_w_validity
         ):
-            logging.info(f"Joining dimension column: {dimension_column}")
-            logging.info(f"df_obs_denorm columns: {df_obs_denorm.columns}")
-            logging.info(
+            logging.debug(f"Joining dimension column: {dimension_column}")
+            logging.debug(f"df_obs_denorm columns: {df_obs_denorm.columns}")
+            logging.debug(
                 f"Is dimension {dimension_column} in the dataframe? {dimension_column in df_obs_denorm.columns}"
             )
-            df_dimension.show(5)
             df_obs_denorm = (
                 df_obs_denorm.alias("o")
                 .join(
@@ -363,9 +362,8 @@ class SWSBronzeIcebergSparkHelper:
                 )
                 .drop("code", "join_id")
             )
-            logging.info(f"After join count: {df_obs_denorm.count()}")
+            logging.debug(f"After join count: {df_obs_denorm.count()}")
 
-        df_element_uom.show(5)
         df_obs_denorm = (
             df_obs_denorm.alias("d")
             .join(
@@ -375,7 +373,7 @@ class SWSBronzeIcebergSparkHelper:
             )
             .drop("element_code")
         )
-        logging.info(f"After uom count: {df_obs_denorm.count()}")
+        logging.debug(f"After uom count: {df_obs_denorm.count()}")
 
         return df_obs_denorm