sws-spark-dissemination-helper 0.0.176__tar.gz → 0.0.185__tar.gz

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.176
+Version: 0.0.185
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer>=0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: sws-api-client==2.7.3
 Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: tzdata==2025.2
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.176"
+version = "0.0.185"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.40.0",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer>=0.11.2",
     "six==1.17.0",
-    "sws_api_client==2.3.0",
+    "sws_api_client==2.7.3",
     "typing_extensions>=4.12.2",
     "tzdata==2025.2",
     "urllib3==1.26.20"

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -8,13 +8,9 @@ from pyspark.sql.functions import col, lit
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import (
-    get_or_create_tag,
-    save_cache_csv,
-    upsert_disseminated_table,
-)
+from .utils import get_or_create_tag, save_cache_csv, upsert_disseminated_table
 
 
 class SWSGoldIcebergSparkHelper:
@@ -66,6 +62,12 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
+            self.display_decimals = (
+                self.sws_postgres_spark_reader.get_display_decimals_datatable(
+                    domain_code=domain_code
+                )
+            )
+
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -86,14 +88,67 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    def keep_dim_val_attr_columns(self, df: DataFrame):
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
         cols_to_keep_sws = self.cols_to_keep_sws
-        if "note" in df.columns:
-            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
         if "unit_of_measure_symbol" in df.columns:
             cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
         return df.select(*cols_to_keep_sws)
 
+    def round_to_display_decimals(self, df: DataFrame):
+        col1_name, col2_name = (
+            self.display_decimals.select("column_1_name", "column_2_name")
+            .distinct()
+            .collect()[0]
+        )
+        if col1_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+        if col2_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+
+        df = (
+            df.alias("d")
+            .join(
+                self.display_decimals.alias("dd"),
+                on=(col(f"d.{col1_name}") == col("dd.column_1_value"))
+                & (col(f"d.{col2_name}") == col("dd.column_2_value")),
+                how="left",
+            )
+            .select("d.*", "dd.display_decimals")
+        )
+
+        df.filter(col("display_decimals").isNull()).select(
+            col1_name, col2_name
+        ).distinct()
+        logging.warning(
+            f"The following combinations of {col1_name} and {col2_name} are not available in the table {DatasetDatatables.DISPLAY_DECIMALS.name} and will be assigned to 0"
+        )
+
+        df = df.withColumn(
+            "display_decimals",
+            F.coalesce(col("display_decimals"), lit("0")).cast("INT"),
+        ).withColumn(
+            "value",
+            F.round(
+                F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
+            )
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
+        )
+
+        # F.round(
+        #     col("value").cast("FLOAT"), col("display_decimals").cast("INT")
+        # ).cast("STRING"),
+
+        return df
+
     def read_bronze_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
@@ -104,18 +159,26 @@ class SWSGoldIcebergSparkHelper:
             self.iceberg_tables.SILVER.iceberg_id
         )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
             self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_data(self) -> DataFrame:
-        return self.read_bronze_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame
@@ -672,29 +735,3 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
-
-
-1
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-1
-1
-2
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-2
-1
-1
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-1
-1
-2
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-2
-1
-1
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-1
-1
-1
-frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
-1
-1

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -497,3 +497,45 @@ class SWSPostgresSparkReader:
                 "aggregation",
             ],
         )
+
+    def get_display_decimals_datatable(
+        self,
+        domain_code: str,
+    ) -> DataFrame:
+        df = self.read_pg_table(
+            pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
+            custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+        ).filter(col("domain") == lit(domain_code))
+
+        pairs = df.select("column_1_name", "column_2_name").distinct().collect()
+
+        # If no config exists for this domain, fail early
+        if not pairs:
+            msg = (
+                f'No display-decimals configuration found for domain "{domain_code}". '
+                f'Please add an entry in table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+            logging.error(msg)
+            # raise ValueError(msg)
+
+        # If more than one mapping exists, it's invalid
+        if len(pairs) > 1:
+            formatted_pairs = [(p["column_1_name"], p["column_2_name"]) for p in pairs]
+
+            msg = (
+                f'Invalid configuration for domain "{domain_code}". '
+                f"Expected exactly one (column_1_name, column_2_name) pair, but found {len(pairs)}: "
+                f"{formatted_pairs}. "
+                f'Please correct the table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+
+            logging.error(
+                "Multiple display-decimals column pairs detected",
+                extra={
+                    "domain": domain_code,
+                    "pairs_found": formatted_pairs,
+                },
+            )
+            raise ValueError(msg)
+
+        return df

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -209,7 +209,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("d.diss_note"),
                             F.concat(
-                                col("sy.diss_note"),
+                                col("sy.note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
@@ -225,7 +225,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("new_diss_note"),
                             F.concat(
-                                col("ey.diss_note"),
+                                col("ey.note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),

{sws_spark_dissemination_helper-0.0.176 → sws_spark_dissemination_helper-0.0.185}/src/sws_spark_dissemination_helper/constants.py

@@ -81,6 +81,19 @@ class DatasetDatatables:
             " method_flag",
         ],
     )
+    DISPLAY_DECIMALS = __SWSDatatable(
+        id="datatables.display_decimals",
+        name="Dissemination - Display Decimals",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, column_1_name STRING, column_1_value STRING, column_2_name STRING, column_2_value STRING, display_decimals STRING",
+        join_columns=[
+            "domain",
+            "column_1_name",
+            "column_1_value",
+            "column_2_name",
+            "column_2_value",
+            "display_decimals",
+        ],
+    )
     # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
@@ -254,9 +267,15 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
-        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
-        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
-        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_observation"
+        )
+        self.DENORMALIZED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_metadata"
+        )
+        self.GROUPED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="grouped_metadata"
+        )
         self.TABLE = self.create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self.create_iceberg_table("BRONZE")