sws-spark-dissemination-helper 0.0.173__tar.gz → 0.0.180__tar.gz

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.173
+Version: 0.0.180
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.173"
+version = "0.0.180"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.40.0",

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -8,13 +8,9 @@ from pyspark.sql.functions import col, lit
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import (
-    get_or_create_tag,
-    save_cache_csv,
-    upsert_disseminated_table,
-)
+from .utils import get_or_create_tag, save_cache_csv, upsert_disseminated_table
 
 
 class SWSGoldIcebergSparkHelper:
@@ -66,6 +62,12 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
+            self.display_decimals = (
+                self.sws_postgres_spark_reader.get_display_decimals_datatable(
+                    domain_code=domain_code
+                )
+            )
+
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -87,7 +89,62 @@ class SWSGoldIcebergSparkHelper:
         return df.filter(col("diss_flag"))
 
     def keep_dim_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sws)
+        cols_to_keep_sws = self.cols_to_keep_sws
+        if "note" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
+
+    def round_to_display_decimals(self, df: DataFrame):
+        col1_name, col2_name = (
+            self.display_decimals.select("column_1_name", "column_2_name")
+            .distinct()
+            .collect()[0]
+        )
+        if col1_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+        if col2_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+
+        df = (
+            df.alias("d")
+            .join(
+                self.display_decimals.alias("dd"),
+                on=(col(f"d.{col1_name}") == col("dd.column_1_value"))
+                & (col(f"d.{col2_name}") == col("dd.column_2_value")),
+                how="left",
+            )
+            .select("d.*", "dd.display_decimals")
+        )
+
+        df.filter(col("display_decimals").isNull()).select(
+            col1_name, col2_name
+        ).distinct()
+        logging.warning(
+            f"The following combinations of {col1_name} and {col2_name} are not available in the table {DatasetDatatables.DISPLAY_DECIMALS.name} and will be assigned to 0"
+        )
+
+        df = df.withColumn(
+            "display_decimals",
+            F.coalesce(col("display_decimals"), lit("0")).cast("INT"),
+        ).withColumn(
+            "value",
+            F.round(
+                F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
+            )
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
+        )
+
+        # F.round(
+        #     col("value").cast("FLOAT"), col("display_decimals").cast("INT")
+        # ).cast("STRING"),
+
+        return df
 
     def read_bronze_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -497,3 +497,45 @@ class SWSPostgresSparkReader:
                 "aggregation",
             ],
         )
+
+    def get_display_decimals_datatable(
+        self,
+        domain_code: str,
+    ) -> DataFrame:
+        df = self.read_pg_table(
+            pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
+            custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+        ).filter(col("domain") == lit(domain_code))
+
+        pairs = df.select("column_1_name", "column_2_name").distinct().collect()
+
+        # If no config exists for this domain, fail early
+        if not pairs:
+            msg = (
+                f'No display-decimals configuration found for domain "{domain_code}". '
+                f'Please add an entry in table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+            logging.error(msg)
+            # raise ValueError(msg)
+
+        # If more than one mapping exists, it's invalid
+        if len(pairs) > 1:
+            formatted_pairs = [(p["column_1_name"], p["column_2_name"]) for p in pairs]
+
+            msg = (
+                f'Invalid configuration for domain "{domain_code}". '
+                f"Expected exactly one (column_1_name, column_2_name) pair, but found {len(pairs)}: "
+                f"{formatted_pairs}. "
+                f'Please correct the table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+
+            logging.error(
+                "Multiple display-decimals column pairs detected",
+                extra={
+                    "domain": domain_code,
+                    "pairs_found": formatted_pairs,
+                },
+            )
+            raise ValueError(msg)
+
+        return df

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -103,7 +103,7 @@ class SWSSilverIcebergSparkHelper:
         # The diss_flag column is needed to initialize the condition expression
         # The note column will contain the eventual reasons why diss_flag has been set to false
         return df.withColumn("diss_flag", lit(True)).withColumn(
-            "note", lit([]).cast(ArrayType(StringType()))
+            "diss_note", lit([]).cast(ArrayType(StringType()))
         )
 
     def read_bronze_data(self) -> DataFrame:
@@ -182,7 +182,7 @@ class SWSSilverIcebergSparkHelper:
                 for column in cols_to_select
                 if column.lower()
                 not in (
-                    "note",
+                    "diss_note",
                     f"{col_name_lower}_start_date",
                     f"{col_name_lower}_end_date",
                 )
@@ -203,36 +203,36 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
                 .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_start_year"),
                         F.array_append(
-                            col("d.note"),
+                            col("d.diss_note"),
                             F.concat(
-                                col("sy.note"),
+                                col("sy.diss_note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
                                 col("sy.new_code"),
                             ),
                         ),
-                    ).otherwise(col("d.note")),
+                    ).otherwise(col("d.diss_note")),
                 )
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_end_year"),
                         F.array_append(
-                            col("new_note"),
+                            col("new_diss_note"),
                             F.concat(
-                                col("ey.note"),
+                                col("ey.diss_note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),
                                 col("ey.new_code"),
                             ),
                         ),
-                    ).otherwise(col("new_note")),
+                    ).otherwise(col("new_diss_note")),
                 )
                 .withColumn(
                     f"new_{col_name}_start_date",
@@ -249,7 +249,7 @@ class SWSSilverIcebergSparkHelper:
                 )
                 .select(
                     *cols_to_select,
-                    col("new_note").alias("note"),
+                    col("new_diss_note").alias("diss_note"),
                     col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),
                     col(f"new_{col_name}_end_date").alias(f"{col_name}_end_date"),
                 )
@@ -270,15 +270,15 @@ class SWSSilverIcebergSparkHelper:
                 start_date_condition & end_date_condition,
             )
             .withColumn("diss_flag", col("diss_flag") & col("condition_result"))
-            # In case the condition is satisfied update diss_flag accordingly and append a note indicating the reason for the observation exclusion from the dissemination
+            # In case the condition is satisfied update diss_flag accordingly and append a diss_note indicating the reason for the observation exclusion from the dissemination
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"), lit(f"{col_type} out of time validity range")
+                        col("diss_note"), lit(f"{col_type} out of time validity range")
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -390,7 +390,7 @@ class SWSSilverIcebergSparkHelper:
             col_name (str): The DataFrame column name on which to apply the filter
 
         Returns:
-            DataFrame: The DataFrame with updated `diss_flag` and `note` columns based on the check outcome
+            DataFrame: The DataFrame with updated `diss_flag` and `diss_note` columns based on the check outcome
         """
 
         # Remove the duplicates that may be in the tables
@@ -428,14 +428,14 @@ class SWSSilverIcebergSparkHelper:
                 col("diss_flag") & col("condition_result"),
             )
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"),
+                        col("diss_note"),
                         lit(f"{col_type} not disseminated for this domain"),
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -522,16 +522,16 @@ class SWSSilverIcebergSparkHelper:
                             col("diss_flag") & col("condition_result"),
                         )
                         .withColumn(
-                            "note",
+                            "diss_note",
                             F.when(
                                 ~col("condition_result"),
                                 F.array_append(
-                                    col("note"),
+                                    col("diss_note"),
                                     lit(
                                         f"not disseminated according to exception with note: {row_exception['note']}"
                                     ),
                                 ),
-                            ).otherwise(col("note")),
+                            ).otherwise(col("diss_note")),
                         )
                         .drop("condition_result")
                     )
@@ -616,7 +616,7 @@ class SWSSilverIcebergSparkHelper:
 
         df = (
             df.withColumn("metadata", F.to_json(col("metadata")))
-            .withColumn("note", F.to_json(col("note")))
+            .withColumn("diss_note", F.to_json(col("diss_note")))
             .coalesce(1)
         )
 
@@ -650,7 +650,7 @@ class SWSSilverIcebergSparkHelper:
                 "value",
                 *self.flag_columns,
                 "diss_flag",
-                "note",
+                "diss_note",
             ],
         )
         tag = upsert_disseminated_table(

{sws_spark_dissemination_helper-0.0.173 → sws_spark_dissemination_helper-0.0.180}/src/sws_spark_dissemination_helper/constants.py

@@ -81,6 +81,19 @@ class DatasetDatatables:
             " method_flag",
         ],
     )
+    DISPLAY_DECIMALS = __SWSDatatable(
+        id="datatables.display_decimals",
+        name="Dissemination - Display Decimals",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, column_1_name STRING, column_1_value STRING, column_2_name STRING, column_2_value STRING, display_decimals STRING",
+        join_columns=[
+            "domain",
+            "column_1_name",
+            "column_1_value",
+            "column_2_name",
+            "column_2_value",
+            "display_decimals",
+        ],
+    )
     # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
@@ -254,9 +267,15 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
-        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
-        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
-        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_observation"
+        )
+        self.DENORMALIZED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_metadata"
+        )
+        self.GROUPED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="grouped_metadata"
+        )
         self.TABLE = self.create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self.create_iceberg_table("BRONZE")