sws-spark-dissemination-helper 0.0.157__tar.gz → 0.0.158__tar.gz

{sws_spark_dissemination_helper-0.0.157 → sws_spark_dissemination_helper-0.0.158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.157
+Version: 0.0.158
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.157 → sws_spark_dissemination_helper-0.0.158}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.157"
+version = "0.0.158"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.157 → sws_spark_dissemination_helper-0.0.158}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -94,25 +94,37 @@ class SWSPostgresSparkReader:
 
             logging.info(f"{pg_table} read start")
 
-            # Read observations from the PostgreSQL table into a DataFrame
-            df = (
-                self.spark.read.format("jdbc")
-                .option("customSchema", custom_schema)
-                .option("dbtable", pg_table)
-                .option("partitionColumn", partition_column)
-                .option("lowerBound", min_id)
-                .option("upperBound", max_id)
-                .option("numPartitions", num_partitions)
-                .option("fetchsize", "1000")
-                .option("url", self.jdbc_url)
-                .option("user", self.jdbc_conn_properties["user"])
-                .option("password", self.jdbc_conn_properties["password"])
-                .option("driver", SPARK_POSTGRES_DRIVER)
-                .load()
-                # .repartition(1024, partition_column)
-                # .sortWithinPartitions(partition_column)
-                # .cache()
-            )
+            if min_id is None or max_id is None:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                )
+            else:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("partitionColumn", partition_column)
+                    .option("lowerBound", min_id)
+                    .option("upperBound", max_id)
+                    .option("numPartitions", num_partitions)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                    # .repartition(1024, partition_column)
+                    # .sortWithinPartitions(partition_column)
+                    # .cache()
+                )
         else:
             df = (
                 self.spark.read.format("jdbc")