sws-spark-dissemination-helper 0.0.175__py3-none-any.whl → 0.0.177__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -8,13 +8,9 @@ from pyspark.sql.functions import col, lit
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import (
-    get_or_create_tag,
-    save_cache_csv,
-    upsert_disseminated_table,
-)
+from .utils import get_or_create_tag, save_cache_csv, upsert_disseminated_table
 
 
 class SWSGoldIcebergSparkHelper:
@@ -66,6 +62,12 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
+            self.display_decimals = (
+                self.sws_postgres_spark_reader.get_display_decimals_datatable(
+                    domain_code=domain_code
+                )
+            )
+
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -87,9 +89,56 @@ class SWSGoldIcebergSparkHelper:
         return df.filter(col("diss_flag"))
 
     def keep_dim_val_attr_columns(self, df: DataFrame):
+        cols_to_keep_sws = self.cols_to_keep_sws
         if "note" in df.columns:
-            return df.select(*self.cols_to_keep_sws, "note")
-        return df.select(*self.cols_to_keep_sws)
+            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
+
+    def add_display_decimals(self, df: DataFrame):
+        col1_name, col2_name = (
+            self.display_decimals.select("column_1_name", "column_2_name")
+            .distinct()
+            .collect()[0]
+        )
+        if col1_name.lower() not in [column.lower() for column in df.columns()]:
+            raise ValueError(
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns()})"
+            )
+        if col2_name.lower() not in [column.lower() for column in df.columns()]:
+            raise ValueError(
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns()})"
+            )
+
+        df = (
+            df.alias("d")
+            .join(
+                self.display_decimals.alias("dd"),
+                on=(col(f"d.{col1_name}") == col("dd.column_1_value"))
+                & (col(f"d.{col2_name}") == col("dd.column_2_value")),
+                how="left",
+            )
+            .select("d.*", "dd.display_decimals")
+        )
+
+        df.filter(col("display_decimals").isNull()).select(
+            col1_name, col2_name
+        ).distinct()
+        logging.warning(
+            f"The following combinations of {col1_name} and {col2_name} are not available in the table {DatasetDatatables.DISPLAY_DECIMALS.name} and will be assigned to 0"
+        )
+
+        df = df.withColumn(
+            "display_decimals", F.coalesce(col("display_decimals"), lit("0"))
+        ).withColumn(
+            "value",
+            F.round(
+                col("value").cast("FLOAT"), col("display_decimals").cast("FLOAT")
+            ).cast("STRING"),
+        )
+
+        return df
 
     def read_bronze_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -497,3 +497,45 @@ class SWSPostgresSparkReader:
                 "aggregation",
             ],
         )
+
+    def get_display_decimals_datatable(
+        self,
+        domain_code: str,
+    ) -> DataFrame:
+        df = self.read_pg_table(
+            pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
+            custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+        ).filter(col("domain") == lit(domain_code))
+
+        pairs = df.select("column_1_name", "column_2_name").distinct().collect()
+
+        # If no config exists for this domain, fail early
+        if not pairs:
+            msg = (
+                f'No display-decimals configuration found for domain "{domain_code}". '
+                f'Please add an entry in table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+            logging.error(msg)
+            # raise ValueError(msg)
+
+        # If more than one mapping exists, it's invalid
+        if len(pairs) > 1:
+            formatted_pairs = [(p["column_1_name"], p["column_2_name"]) for p in pairs]
+
+            msg = (
+                f'Invalid configuration for domain "{domain_code}". '
+                f"Expected exactly one (column_1_name, column_2_name) pair, but found {len(pairs)}: "
+                f"{formatted_pairs}. "
+                f'Please correct the table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+
+            logging.error(
+                "Multiple display-decimals column pairs detected",
+                extra={
+                    "domain": domain_code,
+                    "pairs_found": formatted_pairs,
+                },
+            )
+            raise ValueError(msg)
+
+        return df

sws_spark_dissemination_helper/constants.py

@@ -81,6 +81,19 @@ class DatasetDatatables:
             " method_flag",
         ],
     )
+    DISPLAY_DECIMALS = __SWSDatatable(
+        id="datatables.display_decimals",
+        name="Dissemination - Display Decimals",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, column_1_name STRING, column_1_value STRING, column_2_name STRING, column_2_value STRING, display_decimals STRING",
+        join_columns=[
+            "domain",
+            "column_1_name",
+            "column_1_value",
+            "column_2_name",
+            "column_2_value",
+            "display_decimals",
+        ],
+    )
     # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
@@ -254,9 +267,15 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
-        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
-        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
-        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_observation"
+        )
+        self.DENORMALIZED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_metadata"
+        )
+        self.GROUPED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="grouped_metadata"
+        )
         self.TABLE = self.create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self.create_iceberg_table("BRONZE")

{sws_spark_dissemination_helper-0.0.175.dist-info → sws_spark_dissemination_helper-0.0.177.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.175
+Version: 0.0.177
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.175.dist-info → sws_spark_dissemination_helper-0.0.177.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=CFqNhIn3IKJIFP6efxqGIkQX0xEzC8394XRv2VhkG1w,25143
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=euo9saki8URXF_9J-Y73w847nURlOTJtk6j2DtgB2hQ,27173
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=V_rH4UYoFZfMUc82U-KxeL_o8F44HnMHfLLXoyNxHxs,20016
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=EXpqPHbxld8MPShE6Vo8h4y1vpBt_BjMtS4RPJcPeTU,26355
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
-sws_spark_dissemination_helper/constants.py,sha256=vQmalAqInwPAybgJOfYx99jn47KsKp8jeD8eqmjw-Rs,13471
+sws_spark_dissemination_helper/constants.py,sha256=cVjTS3xbJNKz-1i7c1dJk2PcOZzQhvuHUp9i4PNIPh4,14055
 sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
-sws_spark_dissemination_helper-0.0.175.dist-info/METADATA,sha256=OKWRoPLGWUehzCCgdWqulBhVFtfQuBFw7nzcC9sHcnA,2822
-sws_spark_dissemination_helper-0.0.175.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.175.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.175.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.177.dist-info/METADATA,sha256=9xBCkj_5eIJ9yzuwZJJoqbXK5qbtSncdBmyv5tRNbyU,2822
+sws_spark_dissemination_helper-0.0.177.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.177.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.177.dist-info/RECORD,,