sws-spark-dissemination-helper 0.0.190__tar.gz → 0.0.194__tar.gz

{sws_spark_dissemination_helper-0.0.190 → sws_spark_dissemination_helper-0.0.194}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.190
+Version: 0.0.194
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.190 → sws_spark_dissemination_helper-0.0.194}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.190"
+version = "0.0.194"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.40.0",

{sws_spark_dissemination_helper-0.0.190 → sws_spark_dissemination_helper-0.0.194}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -71,6 +71,9 @@ class SWSGoldIcebergSparkHelper:
             self.display_decimals_df = self.sws_postgres_spark_reader.read_pg_table(
                 pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
                 custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+            ).filter(
+                (col("domain") == lit(self.domain_code))
+                | ((col("domain") == lit("DEFAULT")))
             )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
@@ -110,7 +113,7 @@ class SWSGoldIcebergSparkHelper:
         value_column: str = "value",
     ) -> DataFrame:
 
-        df = df.withColumn("unrounded_value", col(value_column))
+        df = df.withColumn("unrounded_value", col(value_column).cast("string"))
 
         general_default_decimals = (
             self.display_decimals_df.filter(col("domain") == lit("DEFAULT"))
@@ -166,11 +169,15 @@ class SWSGoldIcebergSparkHelper:
             display_decimals = int(rule["display_decimals"])
 
             # Count actual decimal places in the current value
-            # If the value already has fewer decimals than target, skip rounding
+            # Handle both regular decimals and scientific notation
+            # Convert scientific notation to decimal format first
+            value_str_normalized = F.when(
+                F.col(value_column).cast("string").rlike("[eE]"),
+                F.format_number(F.col(value_column).cast("double"), 20),
+            ).otherwise(F.col(value_column).cast("string"))
+
             actual_decimals = F.length(
-                F.regexp_extract(
-                    F.col(value_column).cast("string"), DECIMAL_PLACES_REGEX, 1
-                )
+                F.regexp_extract(value_str_normalized, DECIMAL_PLACES_REGEX, 1)
             )
 
             # Add decimals condition
@@ -184,9 +191,10 @@ class SWSGoldIcebergSparkHelper:
             # Only apply rounding if current decimals >= target decimals
             if display_decimals > 6:
                 # Cast to float and round
-                rounded_value = F.round(
-                    col(value_column).cast(FloatType()), display_decimals
-                )
+                # Cast to DECIMAL with precision 38 and decimals as display_decimals + 2
+                precision = 38
+                decimals = display_decimals
+                rounded_value = col(value_column).cast(DecimalType(precision, decimals))
             else:
                 # Cast to DECIMAL with precision 38 and decimals as display_decimals + 2
                 precision = 38
@@ -234,10 +242,14 @@ class SWSGoldIcebergSparkHelper:
             )
 
         # Only round if actual decimals >= target decimals, otherwise keep original
+        # Handle both regular decimals and scientific notation for default case
+        value_str_normalized_default = F.when(
+            F.col(value_column).cast("string").rlike("[eE]"),
+            F.format_number(F.col(value_column).cast("double"), 20),
+        ).otherwise(F.col(value_column).cast("string"))
+
         actual_decimals_default = F.length(
-            F.regexp_extract(
-                F.col(value_column).cast("string"), DECIMAL_PLACES_REGEX, 1
-            )
+            F.regexp_extract(value_str_normalized_default, DECIMAL_PLACES_REGEX, 1)
         )
         default_rounded = F.when(
             actual_decimals_default >= lit(default_decimals), default_rounded
@@ -840,29 +852,3 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
-
-
-1
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-1
-1
-2
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-2
-1
-1
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-1
-1
-2
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-2
-1
-1
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-1
-1
-1
-frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
-1
-1