sws-spark-dissemination-helper 0.0.60__py3-none-any.whl → 0.0.171__py3-none-any.whl

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -94,25 +94,37 @@ class SWSPostgresSparkReader:
 
             logging.info(f"{pg_table} read start")
 
-            # Read observations from the PostgreSQL table into a DataFrame
-            df = (
-                self.spark.read.format("jdbc")
-                .option("customSchema", custom_schema)
-                .option("dbtable", pg_table)
-                .option("partitionColumn", partition_column)
-                .option("lowerBound", min_id)
-                .option("upperBound", max_id)
-                .option("numPartitions", num_partitions)
-                .option("fetchsize", "1000")
-                .option("url", self.jdbc_url)
-                .option("user", self.jdbc_conn_properties["user"])
-                .option("password", self.jdbc_conn_properties["password"])
-                .option("driver", SPARK_POSTGRES_DRIVER)
-                .load()
-                # .repartition(1024, partition_column)
-                # .sortWithinPartitions(partition_column)
-                # .cache()
-            )
+            if min_id is None or max_id is None:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                )
+            else:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("partitionColumn", partition_column)
+                    .option("lowerBound", min_id)
+                    .option("upperBound", max_id)
+                    .option("numPartitions", num_partitions)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                    # .repartition(1024, partition_column)
+                    # .sortWithinPartitions(partition_column)
+                    # .cache()
+                )
         else:
             df = (
                 self.spark.read.format("jdbc")
@@ -195,6 +207,7 @@ class SWSPostgresSparkReader:
             (dataset_tables.OBSERVATION_COORDINATE, "id", 10),
             (dataset_tables.METADATA, "id", 10),
             (dataset_tables.METADATA_ELEMENT, "metadata", 10),
+            (dataset_tables.TAG_OBSERVATION, "tag", 10),
         ]
         return self._import_tables(data_tables)
 
@@ -209,25 +222,30 @@ class SWSPostgresSparkReader:
             dataset_tables.METADATA_ELEMENT_TYPE,
             dataset_tables.LANGUAGE,
             dataset_tables.UNIT_OF_MEASURE,
+            dataset_tables.DATASET,
             *dataset_tables.CODELISTS,
         ]
+        logging.info(
+            f"Importing reference data tables: {[(table.postgres_id, table.iceberg_id) for table in reference_data_tables]}"
+        )
         return self._import_tables(
             [(table, None, 1) for table in reference_data_tables]
         )
 
     def import_operational_data_tables(
         self, dataset_tables: DatasetTables
-    ) -> DataFrame:
+    ) -> List[DataFrame]:
         # Define and import operational data table without partitioning
         operational_data_tables = [
             (dataset_tables.USER, None, 1),
+            (dataset_tables.TAG, None, 1),
         ]
-        return self._import_tables(operational_data_tables)[0]
+        return self._import_tables(operational_data_tables)
 
     def import_data_reference_data_operational_data(
         self, dataset_tables: DatasetTables
     ) -> Tuple[
-        Tuple[DataFrame, DataFrame, DataFrame, DataFrame],
+        Tuple[DataFrame, DataFrame, DataFrame, DataFrame, DataFrame],
         Tuple[
             DataFrame,
             DataFrame,
@@ -235,22 +253,23 @@ class SWSPostgresSparkReader:
             DataFrame,
             DataFrame,
             DataFrame,
+            DataFrame,
             List[DataFrame],
         ],
-        DataFrame,
+        Tuple[DataFrame, DataFrame],
     ]:
         # Import and organize DataFrames into the desired output structure
         data_dfs = self.import_data_tables(dataset_tables)
         reference_data_dfs = self.import_reference_data_tables(dataset_tables)
-        operational_data_df = self.import_operational_data_tables(dataset_tables)
+        operational_data_dfs = self.import_operational_data_tables(dataset_tables)
 
         return (
             tuple(data_dfs),
             (
-                *reference_data_dfs[:6],
-                reference_data_dfs[6:],
+                *reference_data_dfs[:7],
+                reference_data_dfs[7:],
             ),
-            operational_data_df,
+            tuple(operational_data_dfs),
         )
 
     def get_codelist_type_mapping(
@@ -291,13 +310,73 @@ class SWSPostgresSparkReader:
         self,
         domain_code: str,
     ) -> DataFrame:
-        return self.read_pg_table_and_check_duplicates_for_domain(
+        df = self.read_pg_table(
             pg_table=DatasetDatatables.MAPPING_CODE_CORRECTION.id,
-            table_name=DatasetDatatables.MAPPING_CODE_CORRECTION.name,
             custom_schema=DatasetDatatables.MAPPING_CODE_CORRECTION.schema,
-            domain_code=domain_code,
-            unique_columns=["old_code"],
         )
+        df.filter(
+            col("mapping_type").isNull() | (col("mapping_type") == lit(""))
+        ).transform(
+            correct_domain_filter, domain=domain_code, unique_columns=["old_code"]
+        )
+
+        return df
+
+    def get_domain_code_source_datasets_ids_dest_dataset_id(
+        self, dataset_id: str, domain_code: str = None
+    ) -> Tuple[str, List[str], str]:
+        mapping_domains_id_df = self.read_pg_table(
+            pg_table=DatasetDatatables.MAPPING_DOMAINS_ID.id,
+            custom_schema=DatasetDatatables.MAPPING_DOMAINS_ID.schema,
+        )
+
+        if domain_code is None:
+            domain_code_df = mapping_domains_id_df.filter(
+                col("sws_source_id") == lit(dataset_id)
+            ).select("domain")
+
+            if domain_code_df.count() == 0:
+                raise ValueError(
+                    f'There is no row connecting the current source dataset id ({dataset_id}) to any domain in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+                )
+
+            if domain_code_df.count() > 1:
+                raise ValueError(
+                    f'There is more than one domain referencing the current source dataset id ({dataset_id}) in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}", please specify the domain code you want to process in the parameters'
+                )
+
+            domain_code = domain_code_df.collect()[0][0]
+
+        source_datasets_ids = [
+            row[0]
+            for row in (
+                mapping_domains_id_df.filter(col("domain") == lit(domain_code))
+                .select("sws_source_id")
+                .collect()
+            )
+        ]
+        dest_datasets_id_df = (
+            mapping_domains_id_df.filter(col("domain") == lit(domain_code))
+            .select("sws_destination_id")
+            .distinct()
+        )
+
+        if dest_datasets_id_df.count() == 0:
+            raise ValueError(
+                f'There is no row connecting the current source dataset id and domain pair ({dataset_id}, {domain_code}) to any destination dataset id in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+            )
+        if dest_datasets_id_df.count() > 1:
+            raise ValueError(
+                f'The source dataset id and domain pair ({dataset_id}, {domain_code}) must point only to one destination dataset in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+            )
+
+        dest_datasets_id = dest_datasets_id_df.collect()[0][0]
+
+        logging.info(f"domain code: {domain_code}")
+        logging.info(f"source datasets ids: {source_datasets_ids}")
+        logging.info(f"dest datasets ids: {dest_datasets_id}")
+
+        return (domain_code, source_datasets_ids, dest_datasets_id)
 
     def get_dest_dataset_id(self, domain_code: str, dataset_id: str) -> Tuple[str, str]:
 

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -10,11 +10,11 @@ from pyspark.sql.window import Window
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
 from .utils import (
     get_or_create_tag,
-    map_codes,
+    map_codes_and_remove_null_duplicates,
     save_cache_csv,
     upsert_disseminated_table,
 )
@@ -111,6 +111,11 @@ class SWSSilverIcebergSparkHelper:
             self.iceberg_tables.BRONZE.iceberg_id
         )
 
+    def read_bronze_diss_tag_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
+        )
+
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -158,6 +163,99 @@ class SWSSilverIcebergSparkHelper:
 
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
+        if col_type == "area":
+            logging.info(
+                f'Changing start and end year according to "{DatasetDatatables.MAPPING_CODE_CORRECTION.name}"'
+            )
+            df_start_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("start_year")
+            )
+            df_end_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("end_year")
+            )
+
+            original_col_order = df.columns
+            cols_to_select = df.columns
+            col_name_lower = col_name.lower()
+            cols_to_select = [
+                column
+                for column in cols_to_select
+                if column.lower()
+                not in (
+                    "note",
+                    f"{col_name_lower}_start_date",
+                    f"{col_name_lower}_end_date",
+                )
+            ]
+
+            df = (
+                df.alias("d")
+                .join(
+                    F.broadcast(df_start_year_correction).alias("sy"),
+                    on=col(f"d.{col_name}") == col("sy.mapping_type"),
+                    how="left",
+                )
+                .join(
+                    F.broadcast(df_end_year_correction).alias("ey"),
+                    on=col(f"d.{col_name}") == col("ey.mapping_type"),
+                    how="left",
+                )
+                .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
+                .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
+                .withColumn(
+                    "new_note",
+                    F.when(
+                        col("valid_new_start_year"),
+                        F.array_append(
+                            col("d.note"),
+                            F.concat(
+                                col("sy.note"),
+                                lit(" from "),
+                                col("sy.old_code"),
+                                lit(" to "),
+                                col("sy.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("d.note")),
+                )
+                .withColumn(
+                    "new_note",
+                    F.when(
+                        col("valid_new_end_year"),
+                        F.array_append(
+                            col("new_note"),
+                            F.concat(
+                                col("ey.note"),
+                                lit(" from "),
+                                col("ey.old_code"),
+                                lit(" to "),
+                                col("ey.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("new_note")),
+                )
+                .withColumn(
+                    f"new_{col_name}_start_date",
+                    F.when(
+                        col("valid_new_start_year"), F.to_date(col("sy.new_code"))
+                    ).otherwise(col(f"d.{col_name}_start_date")),
+                )
+                .withColumn(
+                    f"new_{col_name}_end_date",
+                    F.when(
+                        col("valid_new_end_year"),
+                        F.to_date(F.concat(col("ey.new_code"), lit("-12-31"))),
+                    ).otherwise(col(f"d.{col_name}_end_date")),
+                )
+                .select(
+                    *cols_to_select,
+                    col("new_note").alias("note"),
+                    col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),
+                    col(f"new_{col_name}_end_date").alias(f"{col_name}_end_date"),
+                )
+                .select(*original_col_order)
+            )
+
         # Iterate through columns and build conditions dynamically
         start_date_condition = col(f"{col_name}_start_date").isNull() | (
             col(f"{col_name}_start_date") <= col(f"{self.time_column}_start_date")
@@ -204,7 +302,7 @@ class SWSSilverIcebergSparkHelper:
     ) -> DataFrame:
         logging.info(f"Correcting codes for column {col_name} of type {col_type}")
 
-        return map_codes(
+        return map_codes_and_remove_null_duplicates(
             df,
             self.df_mapping_code_correction,
             self.domain_code,
@@ -212,6 +310,12 @@ class SWSSilverIcebergSparkHelper:
             col_type,
             src_column="old_code",
             dest_column="new_code",
+            dimension_columns=[
+                column
+                for column in self.dim_columns_w_time
+                if column not in self.flag_columns
+            ],
+            flag_columns=self.flag_columns,
         )
 
     def apply_code_correction(self, df: DataFrame) -> DataFrame:
@@ -434,7 +538,9 @@ class SWSSilverIcebergSparkHelper:
 
         return df
 
-    def remove_null_duplicates(self, df: DataFrame) -> DataFrame:
+    def check_duplicates(
+        self, df: DataFrame, partition_columns: List[str] = None
+    ) -> DataFrame:
         """
         Removes rows from the DataFrame where the combination of specified dimension columns
         (e.g., 'area', 'element', 'product') is duplicated, and the 'value' column is null.
@@ -468,31 +574,33 @@ class SWSSilverIcebergSparkHelper:
             - The intermediate 'count' column is dropped from the resulting DataFrame.
         """
         # Step 1: Define a window specification based on area, element, and product
-        window_spec = Window.partitionBy(*self.dim_columns_w_time)
-
-        # Step 2: Count the occurrences of each combination of area, element, and product
-        df_with_count = df.withColumn("count", F.count("*").over(window_spec))
-        df_with_count_non_null = df.filter(col("value").isNotNull()).withColumn(
-            "count", F.count("*").over(window_spec)
+        window_spec = Window.partitionBy(
+            *(
+                partition_columns
+                or [
+                    col
+                    for col in self.dim_columns_w_time
+                    if col not in self.flag_columns
+                ]
+            )
         )
 
-        # Step 3: Filter out rows where value is null and the combination occurs more than once and drop the count column as it's no longer needed
-        df_filtered = df_with_count.filter(
-            ~((F.col("value").isNull()) & (F.col("count") > 1))
-        ).drop("count")
-
-        df_warning = df_with_count_non_null.filter(F.col("count") > 1)
-        warning_rows = [row[0] for row in df_warning.select("id").collect()]
-        blocking_warning_rows = df_warning.filter(col("diss_flag")).collect()
-        logging.warning(
-            f"The rows with the following ids are duplicates but they are not disseminated: {str(warning_rows)}"
+        # Step 2: Count the occurrences of each combination of dimensions that is disseminated
+        df_duplicates = (
+            df.filter(col("diss_flag"))
+            .withColumn("count", F.count(lit(1)).over(window_spec))
+            .filter(col("count") > lit(1))
+            .drop("count")
         )
-        if len(blocking_warning_rows) > 0:
+        if df_duplicates.count() > 0:
+            df_duplicates.writeTo(
+                f"{self.iceberg_tables.SILVER.iceberg_id}_duplicates"
+            ).createOrReplace()
             raise RuntimeError(
-                f"There are some duplicates in the data that are flagged for dissemination: {str(blocking_warning_rows)}"
+                f"There are some duplicates in the data that are flagged for dissemination they can be checked in the {self.iceberg_tables.SILVER.iceberg_id}_duplicates table"
             )
 
-        return df_filtered
+        return df
 
     def write_silver_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
 
@@ -537,6 +645,13 @@ class SWSSilverIcebergSparkHelper:
             table=self.iceberg_tables.SILVER.table,
             path=self.iceberg_tables.SILVER.path,
             structure={"columns": df.schema.jsonValue()["fields"]},
+            pinned_columns=[
+                *self.dim_columns_w_time,
+                "value",
+                *self.flag_columns,
+                "diss_flag",
+                "note",
+            ],
         )
         tag = upsert_disseminated_table(
             sws_tags=tags,

sws_spark_dissemination_helper/__init__.py

@@ -2,3 +2,4 @@ from .SWSPostgresSparkReader import SWSPostgresSparkReader
 from .SWSBronzeIcebergSparkHelper import SWSBronzeIcebergSparkHelper
 from .SWSSilverIcebergSparkHelper import SWSSilverIcebergSparkHelper
 from .SWSGoldIcebergSparkHelper import SWSGoldIcebergSparkHelper
+from .SWSEasyIcebergSparkHelper import SWSEasyIcebergSparkHelper

sws_spark_dissemination_helper/constants.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from pyspark.sql.functions import col, lit
 
 SPARK_POSTGRES_DRIVER = "org.postgresql.Driver"
@@ -34,26 +36,57 @@ class DomainFilters:
 class DatasetDatatables:
 
     class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
+        def __init__(
+            self, id: str, name: str, schema: str, join_columns: List[str] = []
+        ):
             self.id = id
+            self.iceberg_id = f"{IcebergDatabases.BRONZE_DATABASE}.{id.split('.')[1]}"
             self.name = name
             self.schema = schema
+            self.join_columns = join_columns
+
+    # Aggregation Tables
+    AGGREGATES_COMPOSITION = __SWSDatatable(
+        id="datatables.aggregates_composition",
+        name="Aggregation - Composition",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, aggregation_type STRING, group_code STRING, child_code STRING, group_name STRING, child_name STRING, link_code STRING, factor STRING",
+    )
+    AGGREGATES_ELEMENTS = __SWSDatatable(
+        id="datatables.aggregates_elements",
+        name="Aggregation - Aggregates per elements",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
+    )
 
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
         name="Dissemination - {type} list",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
     DISSEMINATION_EXCEPTIONS = __SWSDatatable(
         id="datatables.dissemination_exception",
         name="Dissemination - Exceptions",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
+        join_columns=[
+            "domain",
+            " dim1_code",
+            " dim2_code",
+            " dim3_code",
+            " dim4_code",
+            " dim5_code",
+            " dim6_code",
+            " dim7_code",
+            " status_flag",
+            " method_flag",
+        ],
     )
+    # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
         name="Dissemination - Item list - FAOSTAT",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
 
     # Mapping Tables
@@ -61,34 +94,23 @@ class DatasetDatatables:
         id="datatables.aggregates_mapping_domains_id",
         name="Mapping - Domains ID",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
+        join_columns=["domain", "sws_source_id"],
     )
     MAPPING_CODELIST_TYPE = __SWSDatatable(
         id="datatables.mapping_codelist_type",
         name="Mapping Codelist type",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
+        join_columns=["domain", "col_name"],
     )
     MAPPING_CODE_CORRECTION = __SWSDatatable(
         id="datatables.aggregates_mapping_code_correction",
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_SDMX_COLUMN_NAMES = __SWSDatatable(
-        id="datatables.mapping_sdmx_col_names",
-        name="Mapping - SDMX column names",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_name STRING, external_name STRING, delete BOOLEAN, add BOOLEAN, default_value STRING",
-    )
-    MAPPING_SDMX_CODES = __SWSDatatable(
-        id="datatables.mapping_pre_dissemination",
-        name="Mapping - Pre dissemination",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_UNITS_OF_MEASURE = __SWSDatatable(
-        id="datatables.mapping_units_of_measure",
-        name="Mapping - Units of measure",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, sws_code STRING, sws_multiplier INT, sdmx_code STRING, sdmx_multiplier INT, value_multiplier INT, delete BOOLEAN, mapping_type STRING",
+        join_columns=["domain", "old_code", "var_type", "mapping_type"],
     )
 
     # Non-SWS Sources Tables
+    # TODO To deprecate
     FAOSTAT_CODE_MAPPING = __SWSDatatable(
         id="datatables.faostat_code_mapping",
         name="FAOSTAT Code Mapping",
@@ -150,6 +172,11 @@ class DatasetTables:
             iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata_element",
             schema="id BIGINT, metadata INT, metadata_element_type INT, value STRING",
         )
+        self.TAG_OBSERVATION = self.__SWSTable(
+            postgres_id=f"{self.__dataset_id}.tag_observation",
+            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_tag_observation",
+            schema="tag BIGINT, observation INT",
+        )
 
         # Reference data
         self.CODELISTS = [
@@ -181,18 +208,21 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_element_type",
         schema="id INT, metadata_type INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN, private BOOLEAN",
     )
-
     LANGUAGE = __SWSTable(
         postgres_id="reference_data.language",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.language",
         schema="id INT, country_code STRING, description STRING",
     )
-
     UNIT_OF_MEASURE = __SWSTable(
         postgres_id="reference_data.unit_of_measure",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.unit_of_measure",
         schema="id INT, code STRING, sdmx_code STRING, metric BOOLEAN, description STRING, symbol STRING, base_unit STRING, multiplier DECIMAL",
     )
+    DATASET = __SWSTable(
+        postgres_id="reference_data.dataset",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.dataset",
+        schema="id INT, xml_name STRING",
+    )
 
     # Operational data
     USER = __SWSTable(
@@ -200,6 +230,11 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.user",
         schema="id INT, username STRING, preferences INT, email STRING, active BOOLEAN, settings STRING",
     )
+    TAG = __SWSTable(
+        postgres_id="operational_data.tag",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.tag",
+        schema="id INT, name STRING, reference_date DATE, dataset INT, type STRING, released_ON DATE, released_by INT, properties STRING",
+    )
 
 
 class IcebergTable:
@@ -218,21 +253,38 @@ class IcebergTables:
         self.__dataset_id = dataset_id
         self.__tag_name = tag_name
 
-        self.BRONZE = self._create_iceberg_table("BRONZE")
-        self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
+        # TODO Fix later with a more appropriate DATABASE
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.TABLE = self.create_iceberg_table("BRONZE")
+        self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
+        self.BRONZE = self.create_iceberg_table("BRONZE")
+        self.BRONZE_DISS_TAG = self.create_iceberg_table("BRONZE", suffix="diss_tag")
+        self.SILVER = self.create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SDMX = self._create_iceberg_table(
+        self.GOLD_SWS = self.create_iceberg_table("GOLD", prefix=domain, suffix="sws")
+        self.GOLD_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )
-        self.GOLD_SWS_VALIDATED = self._create_iceberg_table(
+        self.GOLD_SWS_VALIDATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_validated"
         )
-        self.GOLD_SWS_DISSEMINATED = self._create_iceberg_table(
+        self.GOLD_SWS_DISSEMINATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_disseminated"
         )
+        self.GOLD_PRE_SDMX = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="pre_sdmx"
+        )
+        self.GOLD_FAOSTAT = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
-    def _create_iceberg_table(
+    def create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""
     ) -> IcebergTable:
         database = getattr(IcebergDatabases, f"{level}_DATABASE")