sws-spark-dissemination-helper 0.0.156__tar.gz → 0.0.158__tar.gz

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.156
+Version: 0.0.158
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.158}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.156"
+version = "0.0.158"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.158}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -347,12 +347,11 @@ class SWSBronzeIcebergSparkHelper:
         for dimension_column, df_dimension in zip(
             self.dim_columns_w_time, dfs_dimension_w_validity
         ):
-            logging.info(f"Joining dimension column: {dimension_column}")
-            logging.info(f"df_obs_denorm columns: {df_obs_denorm.columns}")
-            logging.info(
+            logging.debug(f"Joining dimension column: {dimension_column}")
+            logging.debug(f"df_obs_denorm columns: {df_obs_denorm.columns}")
+            logging.debug(
                 f"Is dimension {dimension_column} in the dataframe? {dimension_column in df_obs_denorm.columns}"
             )
-            df_dimension.show(5)
             df_obs_denorm = (
                 df_obs_denorm.alias("o")
                 .join(
@@ -363,9 +362,8 @@ class SWSBronzeIcebergSparkHelper:
                 )
                 .drop("code", "join_id")
             )
-            logging.info(f"After join count: {df_obs_denorm.count()}")
+            logging.debug(f"After join count: {df_obs_denorm.count()}")
 
-        df_element_uom.show(5)
         df_obs_denorm = (
             df_obs_denorm.alias("d")
             .join(
@@ -375,7 +373,7 @@ class SWSBronzeIcebergSparkHelper:
             )
             .drop("element_code")
         )
-        logging.info(f"After uom count: {df_obs_denorm.count()}")
+        logging.debug(f"After uom count: {df_obs_denorm.count()}")
 
         return df_obs_denorm
 

{sws_spark_dissemination_helper-0.0.156 → sws_spark_dissemination_helper-0.0.158}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -94,25 +94,37 @@ class SWSPostgresSparkReader:
 
             logging.info(f"{pg_table} read start")
 
-            # Read observations from the PostgreSQL table into a DataFrame
-            df = (
-                self.spark.read.format("jdbc")
-                .option("customSchema", custom_schema)
-                .option("dbtable", pg_table)
-                .option("partitionColumn", partition_column)
-                .option("lowerBound", min_id)
-                .option("upperBound", max_id)
-                .option("numPartitions", num_partitions)
-                .option("fetchsize", "1000")
-                .option("url", self.jdbc_url)
-                .option("user", self.jdbc_conn_properties["user"])
-                .option("password", self.jdbc_conn_properties["password"])
-                .option("driver", SPARK_POSTGRES_DRIVER)
-                .load()
-                # .repartition(1024, partition_column)
-                # .sortWithinPartitions(partition_column)
-                # .cache()
-            )
+            if min_id is None or max_id is None:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                )
+            else:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("partitionColumn", partition_column)
+                    .option("lowerBound", min_id)
+                    .option("upperBound", max_id)
+                    .option("numPartitions", num_partitions)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                    # .repartition(1024, partition_column)
+                    # .sortWithinPartitions(partition_column)
+                    # .cache()
+                )
         else:
             df = (
                 self.spark.read.format("jdbc")