sws-spark-dissemination-helper 0.0.191__py3-none-any.whl → 0.0.193__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -71,6 +71,9 @@ class SWSGoldIcebergSparkHelper:
             self.display_decimals_df = self.sws_postgres_spark_reader.read_pg_table(
                 pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
                 custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+            ).filter(
+                (col("domain") == lit(self.domain_code))
+                | ((col("domain") == lit("DEFAULT")))
             )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
@@ -166,11 +169,15 @@ class SWSGoldIcebergSparkHelper:
             display_decimals = int(rule["display_decimals"])
 
             # Count actual decimal places in the current value
-            # If the value already has fewer decimals than target, skip rounding
+            # Handle both regular decimals and scientific notation
+            # Convert scientific notation to decimal format first
+            value_str_normalized = F.when(
+                F.col(value_column).cast("string").rlike("[eE]"),
+                F.format_number(F.col(value_column).cast("double"), 20),
+            ).otherwise(F.col(value_column).cast("string"))
+
             actual_decimals = F.length(
-                F.regexp_extract(
-                    F.col(value_column).cast("string"), DECIMAL_PLACES_REGEX, 1
-                )
+                F.regexp_extract(value_str_normalized, DECIMAL_PLACES_REGEX, 1)
             )
 
             # Add decimals condition
@@ -234,10 +241,14 @@ class SWSGoldIcebergSparkHelper:
             )
 
         # Only round if actual decimals >= target decimals, otherwise keep original
+        # Handle both regular decimals and scientific notation for default case
+        value_str_normalized_default = F.when(
+            F.col(value_column).cast("string").rlike("[eE]"),
+            F.format_number(F.col(value_column).cast("double"), 20),
+        ).otherwise(F.col(value_column).cast("string"))
+
         actual_decimals_default = F.length(
-            F.regexp_extract(
-                F.col(value_column).cast("string"), DECIMAL_PLACES_REGEX, 1
-            )
+            F.regexp_extract(value_str_normalized_default, DECIMAL_PLACES_REGEX, 1)
         )
         default_rounded = F.when(
             actual_decimals_default >= lit(default_decimals), default_rounded
@@ -866,3 +877,27 @@ frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
 frozenset({"1", "2", "6", "7", "5", "8", "0", "4", "3", "9"})
 1
 1
+1
+frozenset({"2", "4", "5", "7", "9", "3", "1", "0", "6", "8"})
+1
+1
+2
+frozenset({"2", "4", "5", "7", "9", "3", "1", "0", "6", "8"})
+2
+1
+1
+frozenset({"2", "4", "5", "7", "9", "3", "1", "0", "6", "8"})
+1
+1
+2
+frozenset({"2", "4", "5", "7", "9", "3", "1", "0", "6", "8"})
+2
+1
+1
+frozenset({"2", "4", "5", "7", "9", "3", "1", "0", "6", "8"})
+1
+1
+1
+frozenset({'2', '4', '5', '7', '9', '3', '1', '0', '6', '8'})
+1
+1

{sws_spark_dissemination_helper-0.0.191.dist-info → sws_spark_dissemination_helper-0.0.193.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.191
+Version: 0.0.193
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.191.dist-info → sws_spark_dissemination_helper-0.0.193.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=3JkM3u7fLG-ZTtuReR4y3q5JVbhv_XG7-faRYQaktc0,32378
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=0LonVBX39-cEzo7tul06YX6G70BVo7zDE_uA2Lu-4Cc,33441
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=5do-Cz2_GAEwNxPWRnnITjADMX8Wgi3aj_ynpQCUNmI,18467
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=PGZPq_oNKRGOseYGuNujbcS8y-WuLmoDMN95faq0Css,26359
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
 sws_spark_dissemination_helper/constants.py,sha256=MzuC7pqsXF89r-FK7hhmWaZSk5x3GB_YPVSfuK3NYVY,14056
 sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
-sws_spark_dissemination_helper-0.0.191.dist-info/METADATA,sha256=GCVYkvlKzxgFc22jEYBEc2_Hj7PN1RJAoamlnXdM4nA,2822
-sws_spark_dissemination_helper-0.0.191.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.191.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.191.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.193.dist-info/METADATA,sha256=ZRxTBrv7G6LjvMrRpDNDEdAxCPIW0hjCmLIvZrE-hY0,2822
+sws_spark_dissemination_helper-0.0.193.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.193.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.193.dist-info/RECORD,,