sws-spark-dissemination-helper 0.0.79__py3-none-any.whl → 0.0.183__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -8,14 +8,9 @@ from pyspark.sql.functions import col, lit
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import (
-    col_is_null_or_empty,
-    get_or_create_tag,
-    save_cache_csv,
-    upsert_disseminated_table,
-)
+from .utils import get_or_create_tag, save_cache_csv, upsert_disseminated_table
 
 
 class SWSGoldIcebergSparkHelper:
@@ -47,11 +42,6 @@ class SWSGoldIcebergSparkHelper:
                 self.flag_columns,
             ) = self._get_dim_time_flag_columns()
 
-            self.cols_to_keep_sdmx = (
-                self.dim_columns_w_time
-                + ["unit_of_measure", "unit_of_measure_multiplier", "value"]
-                + self.flag_columns
-            )
             self.cols_to_keep_sws = (
                 self.dim_columns_w_time + ["value"] + self.flag_columns
             )
@@ -72,41 +62,11 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
-            (
-                self.df_mapping_sdmx_codes,
-                self.df_mapping_sdmx_uom,
-                self.df_mapping_sdmx_col_names,
-            ) = sws_postgres_spark_reader.import_sdmx_mapping_datatables(
-                self.domain_code
-            )
-
-            self._check_column_mappings(self.df_mapping_sdmx_col_names)
-
-    def _check_column_mappings(
-        self,
-        df_mapping_sdmx_col_names: DataFrame,
-    ) -> DataFrame:
-        cols_to_keep_set = set(self.cols_to_keep_sdmx)
-        mapping_sdmx_col_names_internal_set = {
-            row[0]
-            for row in df_mapping_sdmx_col_names.filter(
-                col("internal_name").isNotNull() & (col("internal_name") != lit(""))
-            )
-            .select("internal_name")
-            .collect()
-        }
-
-        if not (cols_to_keep_set <= mapping_sdmx_col_names_internal_set):
-            missing_mappings = cols_to_keep_set - mapping_sdmx_col_names_internal_set
-
-            message = 'The mappings in the table "Mapping - SDMX columns names" are not correct'
-
-            if len(missing_mappings) > 0:
-                message += (
-                    f"\nThe following column mappings are missing: {missing_mappings}"
+            self.display_decimals = (
+                self.sws_postgres_spark_reader.get_display_decimals_datatable(
+                    domain_code=domain_code
                 )
-
-            raise ValueError(message)
+            )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
@@ -128,318 +88,96 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    # TODO implement the delete flag
-    def apply_uom_mapping(
-        self,
-        df: DataFrame,
-    ) -> DataFrame:
-        logging.info("mapping unit of measure for dissemination")
-
-        df = df.withColumn(
-            "official_sws_uom",
-            F.when(
-                col_is_null_or_empty("unit_of_measure_base_unit"),
-                col("unit_of_measure"),
-            ).otherwise(col("unit_of_measure_base_unit")),
-        ).withColumn(
-            "official_sws_multiplier",
-            F.coalesce(F.log10(col("unit_of_measure_multiplier")), lit(0)).cast("int"),
-        )
-
-        delete_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            col("delete")
-            & col_is_null_or_empty("sdmx_code")
-            & col("sdmx_multiplier").isNull()
-            & col("value_multiplier").isNull()
-        )
-
-        generic_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            ~col("delete")
-            & col("sws_multiplier").isNull()
-            & col("sdmx_multiplier").isNull()
-            & (col("value_multiplier") == lit(0))
-        )
-
-        specific_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            ~col("delete")
-            & col("sws_multiplier").isNotNull()
-            & col("sdmx_multiplier").isNotNull()
-        )
-
-        # Apply generic uom mapping
-        df = (
-            df.alias("d")
-            .join(
-                generic_df_uom_mapping.alias("m"),
-                col("d.official_sws_uom") == col("m.sws_code"),
-                "left",
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
+        cols_to_keep_sws = self.cols_to_keep_sws
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
+
+    def round_to_display_decimals(self, df: DataFrame):
+        col1_name, col2_name = (
+            self.display_decimals.select("column_1_name", "column_2_name")
+            .distinct()
+            .collect()[0]
+        )
+        if col1_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col1_name} is not part of the columns available for this dataset ({df.columns})"
+            )
+        if col2_name.lower() not in [column.lower() for column in df.columns]:
+            raise ValueError(
+                f"{col2_name} is not part of the columns available for this dataset ({df.columns})"
             )
-            .select("d.*", col("sdmx_code").alias("generic_sdmx_uom"))
-        )
 
-        # Apply specific uom mapping
         df = (
             df.alias("d")
             .join(
-                specific_df_uom_mapping.alias("m"),
-                (col("d.official_sws_uom") == col("m.sws_code"))
-                & (col("d.official_sws_multiplier") == col("m.sws_multiplier")),
-                "left",
-            )
-            .select(
-                "d.*",
-                col("sdmx_code").alias("specific_sdmx_uom"),
-                col("sdmx_multiplier").alias("specific_sdmx_multiplier"),
-                (col("value") * F.pow(lit(10), col("value_multiplier"))).alias(
-                    "specific_sdmx_value"
-                ),
+                self.display_decimals.alias("dd"),
+                on=(col(f"d.{col1_name}") == col("dd.column_1_value"))
+                & (col(f"d.{col2_name}") == col("dd.column_2_value")),
+                how="left",
             )
+            .select("d.*", "dd.display_decimals")
         )
 
-        # Select the official values according to descending specificity
-        df = (
-            df.withColumn(
-                "unit_of_measure",
-                F.coalesce(
-                    col("specific_sdmx_uom"),
-                    col("generic_sdmx_uom"),
-                    col("official_sws_uom"),
-                ),
-            )
-            .withColumn(
-                "unit_of_measure_multiplier",
-                F.coalesce(
-                    col("specific_sdmx_multiplier"), col("official_sws_multiplier")
-                ),
-            )
-            .withColumn(
-                "value",
-                F.coalesce(col("specific_sdmx_value"), col("value")),
-            )
-            # Remove the columns that were not in the original dataset
-            .drop(
-                col("specific_sdmx_uom"),
-                col("specific_sdmx_multiplier"),
-                col("specific_sdmx_value"),
-                col("generic_sdmx_uom"),
-                col("official_sws_uom"),
-                col("official_sws_multiplier"),
-            )
+        df.filter(col("display_decimals").isNull()).select(
+            col1_name, col2_name
+        ).distinct()
+        logging.warning(
+            f"The following combinations of {col1_name} and {col2_name} are not available in the table {DatasetDatatables.DISPLAY_DECIMALS.name} and will be assigned to 0"
         )
 
-        return df
-
-    def keep_dim_uom_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sdmx)
-
-    def keep_dim_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sws)
-
-    def _apply_sdmx_dimension_codes_mapping_single(
-        self,
-        df: DataFrame,
-        dimension_name: str,
-        dimension_type: str,
-    ) -> DataFrame:
-        logging.info(
-            f"mapping column {dimension_name} of type {dimension_type} for dissemination"
-        )
-        return (
-            df.alias("d")
-            # Join the data with the standard mapping for the specific dimension
-            .join(
-                F.broadcast(
-                    self.df_mapping_sdmx_codes.filter(
-                        (col("domain").isNull() | (col("domain") == lit("")))
-                        & (col("var_type") == lit(dimension_type))
-                        & (
-                            col("mapping_type").isNull()
-                            | (col("mapping_type") == lit(""))
-                        )
-                    )
-                ).alias("m_standard"),
-                col(f"d.{dimension_name}") == col("m_standard.internal_code"),
-                "left",
-            )
-            # Join the data with the domain specific mapping for the specific dimension
-            .join(
-                F.broadcast(
-                    self.df_mapping_sdmx_codes.filter(
-                        (col("domain") == lit(self.domain_code))
-                        & (col("var_type") == lit(dimension_type))
-                        & (
-                            col("mapping_type").isNull()
-                            | (col("mapping_type") == lit(""))
-                        )
-                    )
-                ).alias("m_domain"),
-                col(f"d.{dimension_name}") == col("m_domain.internal_code"),
-                "left",
-            )
-            # Select only the columns we are interested in (this step is optional but recommended for debugging)
-            .select(
-                "d.*",
-                col("m_standard.external_code").alias("standard_external_code"),
-                col("m_standard.delete").alias("standard_delete"),
-                col("m_standard.multiplier").alias("standard_multiplier"),
-                col("m_domain.external_code").alias("domain_specific_external_code"),
-                col("m_domain.delete").alias("domain_specific_delete"),
-                col("m_domain.multiplier").alias("domain_specific_multiplier"),
-            )
-            # Filter out records to delete
-            .filter(
-                # Evaluate first the domain specific flag
-                F.when(
-                    col("domain_specific_delete").isNotNull(),
-                    ~col("domain_specific_delete"),
-                )
-                # Then evaluate the general flag
-                .when(
-                    col("standard_delete").isNotNull(), ~col("standard_delete")
-                ).otherwise(lit(True))
-            )
-            .withColumn(
-                dimension_name,
-                # Evaluate first the domain specific mapping
-                F.when(
-                    col("domain_specific_external_code").isNotNull(),
-                    col("domain_specific_external_code"),
-                )
-                # Then evaluate the general mapping
-                .when(
-                    col("standard_external_code").isNotNull(),
-                    col("standard_external_code"),
-                ).otherwise(col(dimension_name)),
-            )
-            .withColumn(
-                "value",
-                # Multiply first by the domain specific multiplier
-                F.when(
-                    col("domain_specific_multiplier").isNotNull(),
-                    col("value") * col("domain_specific_multiplier"),
-                )
-                # Then multiply by the general multiplier
-                .when(
-                    col("standard_external_code").isNotNull(),
-                    col("value") * col("standard_multiplier"),
-                ).otherwise(col("value")),
-            )
-            # Remove the columns that were not in the original dataset
-            .drop(
-                "standard_external_code",
-                "standard_delete",
-                "standard_multiplier",
-                "domain_specific_external_code",
-                "domain_specific_delete",
-                "domain_specific_multiplier",
+        df = df.withColumn(
+            "display_decimals",
+            F.coalesce(col("display_decimals"), lit("0")).cast("INT"),
+        ).withColumn(
+            "value",
+            F.round(
+                F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
             )
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
         )
 
-    def apply_sdmx_dimension_codes_mapping(self, df: DataFrame) -> DataFrame:
-        logging.info("Mapping codes to comply with SDMX standard")
-        for dimension_name, dimension_type in self.codelist_type_mapping.items():
-            df = df.transform(
-                self._apply_sdmx_dimension_codes_mapping_single,
-                dimension_name=dimension_name,
-                dimension_type=dimension_type,
-            )
-
-        return df
-
-    def drop_non_sdmx_columns(self, df: DataFrame) -> DataFrame:
-        cols_to_drop = [
-            row["internal_name"]
-            for row in self.df_mapping_sdmx_col_names.collect()
-            if row["delete"] is True
-        ]
-        logging.info(f"Dropping non-SDMX columns: {cols_to_drop}")
-        return df.drop(*cols_to_drop)
-
-    def apply_sdmx_column_names_mapping(self, df: DataFrame) -> DataFrame:
-        logging.info("Renaming columns to comply with SDMX standard")
-
-        mapping_sws_col_sdmx_col = {
-            row["internal_name"]: row["external_name"]
-            for row in self.df_mapping_sdmx_col_names.filter(
-                col("internal_name").isNotNull()
-                & (col("internal_name") != lit(""))
-                & ~col("delete")
-            ).collect()
-        }
-
-        logging.info(f"Column names mappings: {mapping_sws_col_sdmx_col}")
-
-        return df.withColumnsRenamed(mapping_sws_col_sdmx_col)
-
-    def add_sdmx_default_columns(self, df: DataFrame) -> DataFrame:
-        col_w_default_value = {
-            row["external_name"]: row["default_value"]
-            for row in self.df_mapping_sdmx_col_names.collect()
-            if row["add"] is True
-        }
-
-        logging.info("Adding SDMX columns with default values")
-
-        for name, default_value in col_w_default_value.items():
-            logging.info(
-                f"Adding SDMX column {name} with default value {default_value}"
-            )
-            df = df.withColumn(name, lit(default_value))
+        # F.round(
+        #     col("value").cast("FLOAT"), col("display_decimals").cast("INT")
+        # ).cast("STRING"),
 
         return df
 
-    def rearrange_sdmx_columns(self, df: DataFrame) -> DataFrame:
-        logging.info(
-            "Rearranging the columns to have the following order: Dimensions, TimeDimension, PrimaryMeasure, Attributes"
+    def read_bronze_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
         )
 
-        get_columns_for_type = lambda df, type: [
-            row[0]
-            for row in df.filter(col("type") == lit(type))
-            .select("external_name")
-            .collect()
-        ]
-
-        df_mapping_sdmx_no_del = self.df_mapping_sdmx_col_names.filter(~col("delete"))
-
-        dimensions = get_columns_for_type(df_mapping_sdmx_no_del, "Dimension")
-        time_dimensions = get_columns_for_type(df_mapping_sdmx_no_del, "TimeDimension")
-        primary_measure = get_columns_for_type(df_mapping_sdmx_no_del, "PrimaryMeasure")
-        attributes = get_columns_for_type(df_mapping_sdmx_no_del, "Attribute")
-
-        logging.info(f"Dimensions: {dimensions}")
-        logging.info(f"Time Dimensions: {time_dimensions}")
-        logging.info(f"Primary Measure: {primary_measure}")
-        logging.info(f"Attributes: {attributes}")
-
-        return df.select(*dimensions, *time_dimensions, *primary_measure, *attributes)
+    def read_silver_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.SILVER.iceberg_id
+        )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.SILVER.iceberg_id)
+            self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.BRONZE.iceberg_id)
-            .transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
         )
 
-    def gen_gold_sdmx_data(self) -> DataFrame:
-        return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.SILVER.iceberg_id)
-            .transform(self.apply_diss_flag_filter)
-            .transform(self.apply_uom_mapping)
-            .transform(self.keep_dim_uom_val_attr_columns)
-            .transform(self.apply_sdmx_dimension_codes_mapping)
-            .transform(self.apply_sdmx_column_names_mapping)
-            .transform(self.add_sdmx_default_columns)
-            .transform(self.rearrange_sdmx_columns)
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
         )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
@@ -468,6 +206,37 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_sws_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        df.writeTo(self.iceberg_tables.GOLD_SWS.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold SWS table written to {self.iceberg_tables.GOLD_SWS.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_SWS.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold SWS tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_SWS.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def gen_and_write_gold_sws_data_to_iceberg_and_csv(self) -> DataFrame:
+        self.df_gold_sws = self.gen_gold_sws_data()
+
+        self.write_gold_sws_data_to_iceberg_and_csv(self.df_gold_sws)
+
+        return self.df_gold_sws
+
     def gen_and_write_gold_sws_validated_data_to_iceberg_and_csv(self) -> DataFrame:
         self.df_gold_sws_validated = self.gen_gold_sws_validated_data()
 
@@ -542,7 +311,7 @@ class SWSGoldIcebergSparkHelper:
         """The expected input to this function is the output of the sws disseminated function"""
         for column in self.dim_columns:
             df = df.withColumn(
-                column, F.regexp_replace(col(column), lit("."), lit("_"))
+                column, F.regexp_replace(col(column), lit("\."), lit("_"))
             )
         df = df.withColumnRenamed("value", "OBS_VALUE").withColumnsRenamed(
             {column: column.upper() for column in df.columns}
@@ -550,14 +319,14 @@ class SWSGoldIcebergSparkHelper:
         df.writeTo(self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id).createOrReplace()
 
         logging.info(
-            f"Gold SDMX table written to {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id}"
+            f"Gold pre-SDMX table written to {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id}"
         )
 
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
         )
 
-        logging.info(f"gold SDMX tag '{self.tag_name}' created")
+        logging.info(f"gold pre-SDMX tag '{self.tag_name}' created")
 
         df_1 = df.coalesce(1)
 
@@ -570,12 +339,59 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
-    def gen_and_write_gold_sdmx_data_to_iceberg_and_csv(self) -> DataFrame:
-        self.df_gold_sdmx = self.gen_gold_sdmx_data()
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
 
-        self.write_gold_sdmx_data_to_iceberg_and_csv(self.df_gold_sdmx)
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
 
-        return self.df_gold_sdmx
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def write_gold_faostat_unfiltered_data_to_iceberg_and_csv(
+        self, df: DataFrame
+    ) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(
+            self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT unfiltered table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT unfiltered tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
 
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
@@ -587,7 +403,7 @@ class SWSGoldIcebergSparkHelper:
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_validated_iceberg",
             name=f"{self.domain_code} gold SWS validated Iceberg",
-            description="Gold table containing all the data unmapped and unfiltered in SWS compatible format",
+            description="Gold table containing all the unfiltered tag data, with code correction appplied, in SWS compatible format",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.ICEBERG,
@@ -608,7 +424,7 @@ class SWSGoldIcebergSparkHelper:
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_validated_csv",
             name=f"{self.domain_code} gold SWS validated csv",
-            description="Gold table containing all the data unmapped and unfiltered in SWS compatible format cached in csv",
+            description="Gold table containing all the unfiltered tag data, with code correction appplied, in SWS compatible format, cached in csv",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.CSV,
@@ -636,7 +452,7 @@ class SWSGoldIcebergSparkHelper:
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_disseminated_iceberg",
             name=f"{self.domain_code} gold SWS disseminated Iceberg",
-            description="Gold table containing all the data mapped and filtered in SWS compatible format",
+            description="Gold table containing only the filtered tag data, with code correction appplied, in SWS compatible format",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.ICEBERG,
@@ -657,7 +473,7 @@ class SWSGoldIcebergSparkHelper:
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_sws_disseminated_csv",
             name=f"{self.domain_code} gold SWS disseminated csv",
-            description="Gold table containing all the data mapped and filtered in SWS compatible format format cached in csv",
+            description="Gold table containing only the filtered tag data, with code correction appplied, in SWS compatible format, cached in csv",
             layer=TableLayer.GOLD,
             private=True,
             type=TableType.CSV,
@@ -733,10 +549,11 @@ class SWSGoldIcebergSparkHelper:
 
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_pre_sdmx_iceberg",
-            name=f"{self.domain_code} gold PRE SDMX Iceberg",
+            name=f"{self.domain_code} gold pre-SDMX Iceberg",
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.ICEBERG,
             database=IcebergDatabases.GOLD_DATABASE,
             table=self.iceberg_tables.GOLD_PRE_SDMX.table,
@@ -754,10 +571,11 @@ class SWSGoldIcebergSparkHelper:
 
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold SDMX csv",
-            description="Gold table containing all the cleaned data in SDMX compatible format cached in csv",
+            name=f"{self.domain_code} gold pre-SDMX csv",
+            description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR and cached in csv",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.CSV,
             path=self.iceberg_tables.GOLD_PRE_SDMX.csv_path,
             structure={"columns": df.schema.jsonValue()["fields"]},
@@ -772,3 +590,148 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_sws_dissemination_tag(self, df: DataFrame, tags: Tags) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_sws_iceberg",
+            name=f"{self.domain_code} gold SWS Iceberg",
+            description="Gold table containing the tag data without any processing",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_SWS.table,
+            path=self.iceberg_tables.GOLD_SWS.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
+            description="Gold table containing the tag data without any processing cached in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_SWS.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_unfiltered_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered Iceberg",
+            description="Gold table containing all the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_csv",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df