sws-spark-dissemination-helper 0.0.93__py3-none-any.whl → 0.0.183__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -8,13 +8,9 @@ from pyspark.sql.functions import col, lit
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import (
-    get_or_create_tag,
-    save_cache_csv,
-    upsert_disseminated_table,
-)
+from .utils import get_or_create_tag, save_cache_csv, upsert_disseminated_table
 
 
 class SWSGoldIcebergSparkHelper:
@@ -66,6 +62,12 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
+            self.display_decimals = (
+                self.sws_postgres_spark_reader.get_display_decimals_datatable(
+                    domain_code=domain_code
+                )
+            )
+
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -86,23 +88,97 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    def keep_dim_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sws)
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
+        cols_to_keep_sws = self.cols_to_keep_sws
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
+
+    def round_to_display_decimals(self, df: DataFrame):
+        col1_name, col2_name = (
+            self.display_decimals.select("column_1_name", "column_2_name")
+            .distinct()
+            .collect()[0]
+        )
+        if col1_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+        if col2_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+
+        df = (
+            df.alias("d")
+            .join(
+                self.display_decimals.alias("dd"),
+                on=(col(f"d.{col1_name}") == col("dd.column_1_value"))
+                & (col(f"d.{col2_name}") == col("dd.column_2_value")),
+                how="left",
+            )
+            .select("d.*", "dd.display_decimals")
+        )
+
+        df.filter(col("display_decimals").isNull()).select(
+            col1_name, col2_name
+        ).distinct()
+        logging.warning(
+            f"The following combinations of {col1_name} and {col2_name} are not available in the table {DatasetDatatables.DISPLAY_DECIMALS.name} and will be assigned to 0"
+        )
+
+        df = df.withColumn(
+            "display_decimals",
+            F.coalesce(col("display_decimals"), lit("0")).cast("INT"),
+        ).withColumn(
+            "value",
+            F.round(
+                F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
+            )
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
+        )
+
+        # F.round(
+        #     col("value").cast("FLOAT"), col("display_decimals").cast("INT")
+        # ).cast("STRING"),
+
+        return df
+
+    def read_bronze_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
+        )
 
     def read_silver_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.SILVER.iceberg_id
         )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
             self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
+
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame
@@ -130,6 +206,37 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_sws_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        df.writeTo(self.iceberg_tables.GOLD_SWS.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold SWS table written to {self.iceberg_tables.GOLD_SWS.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_SWS.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold SWS tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_SWS.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def gen_and_write_gold_sws_data_to_iceberg_and_csv(self) -> DataFrame:
+        self.df_gold_sws = self.gen_gold_sws_data()
+
+        self.write_gold_sws_data_to_iceberg_and_csv(self.df_gold_sws)
+
+        return self.df_gold_sws
+
     def gen_and_write_gold_sws_validated_data_to_iceberg_and_csv(self) -> DataFrame:
         self.df_gold_sws_validated = self.gen_gold_sws_validated_data()
 
@@ -232,6 +339,60 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def write_gold_faostat_unfiltered_data_to_iceberg_and_csv(
+        self, df: DataFrame
+    ) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(
+            self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT unfiltered table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT unfiltered tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:
@@ -242,7 +403,7 @@ class SWSGoldIcebergSparkHelper:
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_validated_iceberg",
             name=f"{self.domain_code} gold SWS validated Iceberg",
-            description="Gold table containing all the data unmapped and unfiltered in SWS compatible format",
+            description="Gold table containing all the unfiltered tag data, with code correction appplied, in SWS compatible format",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.ICEBERG,
@@ -263,7 +424,7 @@ class SWSGoldIcebergSparkHelper:
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_validated_csv",
             name=f"{self.domain_code} gold SWS validated csv",
-            description="Gold table containing all the data unmapped and unfiltered in SWS compatible format cached in csv",
+            description="Gold table containing all the unfiltered tag data, with code correction appplied, in SWS compatible format, cached in csv",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.CSV,
@@ -291,7 +452,7 @@ class SWSGoldIcebergSparkHelper:
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_disseminated_iceberg",
             name=f"{self.domain_code} gold SWS disseminated Iceberg",
-            description="Gold table containing all the data mapped and filtered in SWS compatible format",
+            description="Gold table containing only the filtered tag data, with code correction appplied, in SWS compatible format",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.ICEBERG,
@@ -312,7 +473,7 @@ class SWSGoldIcebergSparkHelper:
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_disseminated_csv",
             name=f"{self.domain_code} gold SWS disseminated csv",
-            description="Gold table containing all the data mapped and filtered in SWS compatible format format cached in csv",
+            description="Gold table containing only the filtered tag data, with code correction appplied, in SWS compatible format, cached in csv",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.CSV,
@@ -392,6 +553,7 @@ class SWSGoldIcebergSparkHelper:
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.ICEBERG,
             database=IcebergDatabases.GOLD_DATABASE,
             table=self.iceberg_tables.GOLD_PRE_SDMX.table,
@@ -413,6 +575,7 @@ class SWSGoldIcebergSparkHelper:
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR and cached in csv",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.CSV,
             path=self.iceberg_tables.GOLD_PRE_SDMX.csv_path,
             structure={"columns": df.schema.jsonValue()["fields"]},
@@ -427,3 +590,148 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_sws_dissemination_tag(self, df: DataFrame, tags: Tags) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_sws_iceberg",
+            name=f"{self.domain_code} gold SWS Iceberg",
+            description="Gold table containing the tag data without any processing",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_SWS.table,
+            path=self.iceberg_tables.GOLD_SWS.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
+            description="Gold table containing the tag data without any processing cached in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_SWS.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_unfiltered_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered Iceberg",
+            description="Gold table containing all the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_csv",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -94,25 +94,37 @@ class SWSPostgresSparkReader:
 
             logging.info(f"{pg_table} read start")
 
-            # Read observations from the PostgreSQL table into a DataFrame
-            df = (
-                self.spark.read.format("jdbc")
-                .option("customSchema", custom_schema)
-                .option("dbtable", pg_table)
-                .option("partitionColumn", partition_column)
-                .option("lowerBound", min_id)
-                .option("upperBound", max_id)
-                .option("numPartitions", num_partitions)
-                .option("fetchsize", "1000")
-                .option("url", self.jdbc_url)
-                .option("user", self.jdbc_conn_properties["user"])
-                .option("password", self.jdbc_conn_properties["password"])
-                .option("driver", SPARK_POSTGRES_DRIVER)
-                .load()
-                # .repartition(1024, partition_column)
-                # .sortWithinPartitions(partition_column)
-                # .cache()
-            )
+            if min_id is None or max_id is None:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                )
+            else:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("partitionColumn", partition_column)
+                    .option("lowerBound", min_id)
+                    .option("upperBound", max_id)
+                    .option("numPartitions", num_partitions)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                    # .repartition(1024, partition_column)
+                    # .sortWithinPartitions(partition_column)
+                    # .cache()
+                )
         else:
             df = (
                 self.spark.read.format("jdbc")
@@ -195,6 +207,7 @@ class SWSPostgresSparkReader:
             (dataset_tables.OBSERVATION_COORDINATE, "id", 10),
             (dataset_tables.METADATA, "id", 10),
             (dataset_tables.METADATA_ELEMENT, "metadata", 10),
+            (dataset_tables.TAG_OBSERVATION, "tag", 10),
         ]
         return self._import_tables(data_tables)
 
@@ -209,25 +222,30 @@ class SWSPostgresSparkReader:
             dataset_tables.METADATA_ELEMENT_TYPE,
             dataset_tables.LANGUAGE,
             dataset_tables.UNIT_OF_MEASURE,
+            dataset_tables.DATASET,
             *dataset_tables.CODELISTS,
         ]
+        logging.info(
+            f"Importing reference data tables: {[(table.postgres_id, table.iceberg_id) for table in reference_data_tables]}"
+        )
         return self._import_tables(
             [(table, None, 1) for table in reference_data_tables]
         )
 
     def import_operational_data_tables(
         self, dataset_tables: DatasetTables
-    ) -> DataFrame:
+    ) -> List[DataFrame]:
         # Define and import operational data table without partitioning
         operational_data_tables = [
             (dataset_tables.USER, None, 1),
+            (dataset_tables.TAG, None, 1),
         ]
-        return self._import_tables(operational_data_tables)[0]
+        return self._import_tables(operational_data_tables)
 
     def import_data_reference_data_operational_data(
         self, dataset_tables: DatasetTables
     ) -> Tuple[
-        Tuple[DataFrame, DataFrame, DataFrame, DataFrame],
+        Tuple[DataFrame, DataFrame, DataFrame, DataFrame, DataFrame],
         Tuple[
             DataFrame,
             DataFrame,
@@ -235,22 +253,23 @@ class SWSPostgresSparkReader:
             DataFrame,
             DataFrame,
             DataFrame,
+            DataFrame,
             List[DataFrame],
         ],
-        DataFrame,
+        Tuple[DataFrame, DataFrame],
     ]:
         # Import and organize DataFrames into the desired output structure
         data_dfs = self.import_data_tables(dataset_tables)
         reference_data_dfs = self.import_reference_data_tables(dataset_tables)
-        operational_data_df = self.import_operational_data_tables(dataset_tables)
+        operational_data_dfs = self.import_operational_data_tables(dataset_tables)
 
         return (
             tuple(data_dfs),
             (
-                *reference_data_dfs[:6],
-                reference_data_dfs[6:],
+                *reference_data_dfs[:7],
+                reference_data_dfs[7:],
             ),
-            operational_data_df,
+            tuple(operational_data_dfs),
         )
 
     def get_codelist_type_mapping(
@@ -291,13 +310,17 @@ class SWSPostgresSparkReader:
         self,
         domain_code: str,
     ) -> DataFrame:
-        return self.read_pg_table_and_check_duplicates_for_domain(
+        df = self.read_pg_table(
             pg_table=DatasetDatatables.MAPPING_CODE_CORRECTION.id,
-            table_name=DatasetDatatables.MAPPING_CODE_CORRECTION.name,
             custom_schema=DatasetDatatables.MAPPING_CODE_CORRECTION.schema,
-            domain_code=domain_code,
-            unique_columns=["old_code"],
         )
+        df.filter(
+            col("mapping_type").isNull() | (col("mapping_type") == lit(""))
+        ).transform(
+            correct_domain_filter, domain=domain_code, unique_columns=["old_code"]
+        )
+
+        return df
 
     def get_domain_code_source_datasets_ids_dest_dataset_id(
         self, dataset_id: str, domain_code: str = None
@@ -474,3 +497,45 @@ class SWSPostgresSparkReader:
                 "aggregation",
             ],
         )
+
+    def get_display_decimals_datatable(
+        self,
+        domain_code: str,
+    ) -> DataFrame:
+        df = self.read_pg_table(
+            pg_table=DatasetDatatables.DISPLAY_DECIMALS.id,
+            custom_schema=DatasetDatatables.DISPLAY_DECIMALS.schema,
+        ).filter(col("domain") == lit(domain_code))
+
+        pairs = df.select("column_1_name", "column_2_name").distinct().collect()
+
+        # If no config exists for this domain, fail early
+        if not pairs:
+            msg = (
+                f'No display-decimals configuration found for domain "{domain_code}". '
+                f'Please add an entry in table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+            logging.error(msg)
+            # raise ValueError(msg)
+
+        # If more than one mapping exists, it's invalid
+        if len(pairs) > 1:
+            formatted_pairs = [(p["column_1_name"], p["column_2_name"]) for p in pairs]
+
+            msg = (
+                f'Invalid configuration for domain "{domain_code}". '
+                f"Expected exactly one (column_1_name, column_2_name) pair, but found {len(pairs)}: "
+                f"{formatted_pairs}. "
+                f'Please correct the table "{DatasetDatatables.DISPLAY_DECIMALS.id}".'
+            )
+
+            logging.error(
+                "Multiple display-decimals column pairs detected",
+                extra={
+                    "domain": domain_code,
+                    "pairs_found": formatted_pairs,
+                },
+            )
+            raise ValueError(msg)
+
+        return df