sws-spark-dissemination-helper 0.0.86__tar.gz → 0.0.88__tar.gz

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/.gitignore

@@ -2,6 +2,7 @@
 # You should customize this list as applicable to your project.
 # Learn more about .gitignore:
 #     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
+.*
 
 # Node artifact files
 node_modules/

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.86
+Version: 0.0.88
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.86"
+version = "0.0.88"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -443,3 +443,78 @@ class SWSBronzeIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Bronze Dissemination tags successfully written")
+
+    def write_bronze_disseminated_tag_data_to_iceberg_and_csv(
+        self, dimensions: Dict[str, List[str]]
+    ) -> DataFrame:
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
+        )
+
+        for dimension_name, codes in dimensions.items():
+            if len(codes) != 0:
+                not_in_codes = ",".join([f"'{code}'" for code in codes])
+                self.spark.sql(
+                    f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
+                )
+        disseminated_tag_df = self.spark.read.option("branch", self.tag_name).table(
+            self.iceberg_tables.BRONZE.iceberg_id
+        )
+
+        disseminated_tag_df = disseminated_tag_df.withColumn(
+            "metadata", F.to_json(col("metadata"))
+        ).coalesce(1)
+
+        save_cache_csv(
+            df=disseminated_tag_df,
+            bucket=self.bucket,
+            prefix=f"{self.iceberg_tables.BRONZE.csv_prefix}_disseminated_tag",
+            tag_name=self.tag_name,
+        )
+
+        return disseminated_tag_df
+
+    def write_bronze_sws_filtered_disseminated_tag(self, tags: Tags):
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_bronze_disseminated_tag_iceberg",
+            name=f"{self.domain_code} bronze disseminated tag Iceberg",
+            description="Bronze table containing the raw data imported from the SWS, denormalized and filtered per dimension",
+            layer=TableLayer.BRONZE,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.BRONZE_DATABASE,
+            table=self.iceberg_tables.BRONZE.table,
+            path=self.iceberg_tables.BRONZE.path,
+            structure={
+                "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
+            },
+        )
+        tag = tags.add_dissemination_table(
+            self.dataset_id, self.tag_name, new_iceberg_table
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_csv_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_bronze_disseminated_tag_csv",
+            name=f"{self.domain_code} bronze disseminated tag csv",
+            description="Bronze table containing the raw data imported from the SWS, denormalized and filtered per dimension cached in csv",
+            layer=TableLayer.BRONZE,
+            private=True,
+            type=TableType.CSV,
+            # TODO Correct the path in the origin library
+            path=self.iceberg_tables.BRONZE.csv_path,
+            structure={
+                "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
+            },
+        )
+        tag = tags.add_dissemination_table(
+            self.dataset_id, self.tag_name, new_csv_table
+        )
+
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        logging.info("Bronze Disseminated tag with selection successfully written")

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -89,20 +89,20 @@ class SWSGoldIcebergSparkHelper:
     def keep_dim_val_attr_columns(self, df: DataFrame):
         return df.select(*self.cols_to_keep_sws)
 
+    def read_silver_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.SILVER.iceberg_id
+        )
+
     def gen_gold_sws_disseminated_data(self) -> DataFrame:
         return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.SILVER.iceberg_id)
+            self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
             .transform(self.keep_dim_val_attr_columns)
         )
 
     def gen_gold_sws_validated_data(self) -> DataFrame:
-        return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.BRONZE.iceberg_id)
-            .transform(self.keep_dim_val_attr_columns)
-        )
+        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame

{sws_spark_dissemination_helper-0.0.86 → sws_spark_dissemination_helper-0.0.88}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -110,6 +110,10 @@ class SWSSilverIcebergSparkHelper:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.BRONZE.iceberg_id
         )
+    def read_bronze_diss_tag_data(self) -> DataFrame:
+        return self.spark.read.option("branch", f"diss_tag_{self.tag_name}").table(
+            self.iceberg_tables.BRONZE.iceberg_id
+        )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""

sws_spark_dissemination_helper-0.0.86/.pipeline/bitbucket-pipelines.yml

@@ -1,69 +0,0 @@
-image: python:3.12  # Choose the base image with Python
-
-pipelines:
-  default: 
-    - step:
-        name: Run Tests
-        caches:
-          - pip
-        script:
-          - curl -sSL https://install.python-poetry.org | python3 -  # Install Poetry
-          - export PATH="$HOME/.local/bin:$PATH"  # Add Poetry to PATH
-          - poetry install  # Install your dependencies
-          - poetry run pytest --cov=sws_api_client --cov-report=xml # Run your tests with coverage
-
-  branches:
-    main:
-      - step:
-          name: Run Tests and Version
-          caches:
-            - pip
-          script:
-            - curl -sSL https://install.python-poetry.org | python3 -  # Install Poetry
-            - export PATH="$HOME/.local/bin:$PATH"  # Add Poetry to PATH
-            - poetry install  # Install your dependencies
-            - poetry run pytest  # Run your tests
-            - export BB_TOKEN=$BITBUCKET_APP_PASSWORD  # Set Bitbucket token
-            - export BB_USERNAME=$BITBUCKET_USERNAME  # Set Bitbucket username
-            - poetry run semantic-release version --no-vcs-release # Run semantic release
-
-    feature/*:
-      - step:
-          name: Run Tests and Version
-          caches:
-            - pip
-          script:
-            - curl -sSL https://install.python-poetry.org | python3 -  # Install Poetry
-            - export PATH="$HOME/.local/bin:$PATH"  # Add Poetry to PATH
-            - poetry install  # Install your dependencies
-            - poetry run pytest  # Run your tests
-            - export BB_TOKEN=$BITBUCKET_APP_PASSWORD  # Set Bitbucket token
-            - export BB_USERNAME=$BITBUCKET_USERNAME  # Set Bitbucket username
-            - poetry run semantic-release version --no-vcs-release --no-tag # Run semantic release
-
-    dev:
-      - step:
-          name: Run Tests and Version
-          caches:
-            - pip
-          script:
-            - curl -sSL https://install.python-poetry.org | python3 -  # Install Poetry
-            - export PATH="$HOME/.local/bin:$PATH"  # Add Poetry to PATH
-            - poetry install  # Install your dependencies
-            - poetry run pytest  # Run your tests
-            - export BB_TOKEN=$BITBUCKET_APP_PASSWORD  # Set Bitbucket token
-            - export BB_USERNAME=$BITBUCKET_USERNAME  # Set Bitbucket username
-            - poetry run semantic-release version --no-vcs-release --no-tag # Run semantic release
-
-  tags:
-    v*.*.*:  # Trigger on version tags like v1.0.0, v2.1.0, etc.
-      - step:
-          name: Publish to PyPI
-          caches:
-            - pip
-          script:
-            - curl -sSL https://install.python-poetry.org | python3 -  # Install Poetry
-            - export PATH="$HOME/.local/bin:$PATH"  # Add Poetry to PATH
-            - poetry install --no-dev  # Install dependencies without dev dependencies
-            - poetry config pypi-token.pypi $PYPI_TOKEN  # Set the PyPI token
-            - poetry publish --build  # Build and publish the package to PyPI. force is needed because I already uploaded pypi version till 0.0.7

sws_spark_dissemination_helper-0.0.86/.pipeline/pyproject.toml

@@ -1,34 +0,0 @@
-[tool.poetry]
-name = "sws-spark-dissemination-helper"
-version = "0.0.49"
-description = "A Python helper package providing streamlined Spark functions for efficient data dissemination processes"
-authors = ["Daniele Mansillo <danielemansillo@gmail.com>"]
-license = "MIT"
-readme = "README.md"
-
-[tool.poetry.dependencies]
-python = "^3.8"
-annotated-types = "0.7.0"
-boto3 = "1.34.147"
-botocore = "1.34.147"
-certifi = "2024.7.4"
-charset-normalizer = "3.3.2"
-idna = "3.7"
-jmespath = "1.0.1"
-py4j = "0.10.9.7"
-pydantic = "2.8.2"
-pydantic-core = "2.20.1"
-pyspark = "3.5.1"
-python-dateutil = "2.9.0.post0"
-python-dotenv = "1.0.1"
-requests = "2.32.3"
-s3transfer = "0.10.2"
-six = "1.16.0"
-sws-api-client = "1.0.7b0"
-typing-extensions = "4.12.2"
-urllib3 = "1.26.19"
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

sws_spark_dissemination_helper-0.0.86/.python-version

@@ -1 +0,0 @@
-3.9.20

sws_spark_dissemination_helper-0.0.86/.wip/SWSBaseIcebergSparkHelper.py

@@ -1,129 +0,0 @@
-import logging
-from copy import copy
-from typing import List, Tuple
-
-import pyspark.sql.functions as F
-from pyspark.sql import DataFrame, SparkSession
-from pyspark.sql.functions import col, lit
-from sws_api_client import Tags
-from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
-
-from .constants import IcebergDatabases, IcebergTables, IcebergTable
-from .SWSPostgresSparkReader import SWSPostgresSparkReader
-from .utils import get_or_create_tag, save_cache_csv
-import boto3
-
-
-class SWSBaseIcebergSparkHelper:
-
-    def _write_to_iceberg(
-        self,
-        df: DataFrame,
-        iceberg_table: IcebergTable,
-        tag_name: str,
-    ) -> DataFrame:
-        # Write to Iceberg
-        df.writeTo(iceberg_table.iceberg_id).createOrReplace()
-        logging.info(f"Table written to {iceberg_table.iceberg_id}")
-
-        # Create a tag in Iceberg if required
-        self.spark.sql(
-            f"ALTER TABLE {iceberg_table.iceberg_id} CREATE TAG `{tag_name}`"
-        )
-        logging.info(f"Tag '{tag_name}' created for {iceberg_table.iceberg_id}")
-
-        return df
-
-    def _write_to_csv(
-        self, df: DataFrame, iceberg_table: IcebergTable, bucket: str, tag_name: str
-    ) -> DataFrame:
-        s3 = boto3.client("s3")
-
-        latest_path = f"s3://{bucket}/{iceberg_table.csv_prefix}/latest"
-        tag_path = f"s3://{bucket}/{iceberg_table.csv_prefix}/{tag_name}"
-
-        latest_prefix = f"{iceberg_table.csv_prefix}/latest"
-        tag_prefix = f"{iceberg_table.csv_prefix}/{tag_name}"
-
-        s3.delete_object(Bucket=bucket, Key=f"{latest_prefix}.csv")
-        df.coalesce(1).write.option("header", True).mode("overwrite").csv(latest_path)
-
-        response = s3.list_objects_v2(Bucket=bucket, Prefix=latest_prefix)
-
-        s3_path_objects_keys = [
-            content["Key"] for content in response.get("Contents", {})
-        ]
-        s3_path_csv = [
-            s3_object
-            for s3_object in s3_path_objects_keys
-            if s3_object.endswith(".csv")
-        ][0]
-
-        # Extract the csv from the folder and delete the folder
-        result_latest = s3.copy_object(
-            Bucket=bucket,
-            CopySource={"Bucket": bucket, "Key": s3_path_csv},
-            Key=f"{latest_prefix}.csv",
-        )
-        logging.info(f"Updated latest version of cached csv at {latest_path}.csv")
-
-        result_tag = s3.copy_object(
-            Bucket=bucket,
-            CopySource={"Bucket": bucket, "Key": s3_path_csv},
-            Key=f"{tag_prefix}.csv",
-        )
-        logging.info(f"Wrote the tag version of cached csv at {tag_path}.csv")
-
-        for object in s3_path_objects_keys:
-            s3.delete_object(Bucket=bucket, Key=object)
-        logging.debug("Cleaning the temporary folder of the csv files")
-
-        return df
-
-    def _create_dissemination_tag(
-        self,
-        df: DataFrame,
-        tags: Tags,
-        iceberg_table: IcebergTable,
-        level: str,
-        domain_code: str,
-        iceberg_description: str,
-        csv_description: str,
-    ) -> Tags:
-        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
-        logging.debug(f"Initial Tag: {tag}")
-
-        # Common table structure
-        base_table_structure = {
-            "layer": level,
-            "private": True,
-            "structure": {"columns": df.schema.jsonValue()["fields"]},
-        }
-
-        # Add Iceberg table to tag
-        iceberg_table = BaseDisseminatedTagTable(
-            id=f"{domain_code}_{iceberg_table.level}_iceberg",
-            name=f"{domain_code} {iceberg_table.level} Iceberg",
-            description=iceberg_description,
-            type=TableType.ICEBERG,
-            database=iceberg_table.database,
-            table=iceberg_table.table,
-            path=iceberg_table.path,
-            **base_table_structure,
-        )
-        tags.add_dissemination_table(self.dataset_id, self.tag_name, iceberg_table)
-        logging.debug(f"Tag with added Iceberg Table: {tag}")
-
-        # Add CSV table to tag
-        csv_table = BaseDisseminatedTagTable(
-            id=f"{domain_code}_{iceberg_table.level}_csv",
-            name=f"{domain_code} {iceberg_table.level} CSV",
-            description=csv_description,
-            type=TableType.CSV,
-            path=iceberg_table.csv_path,
-            **base_table_structure,
-        )
-        tags.add_dissemination_table(self.dataset_id, self.tag_name, csv_table)
-        logging.debug(f"Tag with added CSV Table: {tag}")
-
-        return tag

sws_spark_dissemination_helper-0.0.86/.wip/model.py

@@ -1,263 +0,0 @@
-class IcebergCatalog:
-    def __init__(self, id: str, bucket: str) -> None:
-        self.id = "AwsDataCatalog"
-        self.bucket = bucket
-
-
-class IcebergDatabase:
-    def __init__(self, catalog: IcebergCatalog, id: str, level: str) -> None:
-        self.catalog = catalog
-        self.level = level
-        self.id = id
-        self.database = f"{catalog.id}.{id}"
-
-
-class IcebergTable:
-    def __init__(self, database: IcebergDatabase, id) -> None:
-        self.database = database
-        self.id = id
-        self.iceberg_id = f"{database.catalog.id}.{database.id}.{id}"
-        self.path = f"{database.id}"
-        self.csv_prefix = csv_path.rsplit("/", 1)[0]
-        self.csv_path = csv_path
-
-
-class IcebergDatabases:
-
-    STAGING_SCHEME = "sws_dissemination_tags_bronze"
-    BRONZE_SCHEME = "sws_dissemination_tags_bronze"
-    SILVER_SCHEME = "sws_dissemination_tags_silver"
-    GOLD_SCHEME = "sws_dissemination_tags_gold"
-    STAGING_DATABASE = f"{CATALOG}.{STAGING_SCHEME}"
-    BRONZE_DATABASE = f"{CATALOG}.{BRONZE_SCHEME}"
-    SILVER_DATABASE = f"{CATALOG}.{SILVER_SCHEME}"
-    GOLD_DATABASE = f"{CATALOG}.{GOLD_SCHEME}"
-
-
-class DomainFilters:
-    GENERIC = lambda domain_code: (
-        col("domain").isNull()
-        | (col("domain") == lit(""))
-        | (col("domain") == lit(domain_code))
-    )
-    MATCH = lambda domain_code: (col("domain") == lit(domain_code))
-    EMPTY = lambda: (col("domain").isNull() | (col("domain") == lit("")))
-
-
-class DatasetDatatables:
-
-    class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
-            self.id = id
-            self.name = name
-            self.schema = schema
-
-    # Dissemination Tables
-    DISSEMINATION_TYPE_LIST = __SWSDatatable(
-        id="datatables.dissemination_{type}_list",
-        name="Dissemination - {type} list",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
-    )
-    DISSEMINATION_EXCEPTIONS = __SWSDatatable(
-        id="datatables.dissemination_exception",
-        name="Dissemination - Exceptions",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
-    )
-    DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
-        id="datatables.dissemination_item_list_faostat",
-        name="Dissemination - Item list - FAOSTAT",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
-    )
-
-    # Mapping Tables
-    MAPPING_DOMAINS_ID = __SWSDatatable(
-        id="datatables.aggregates_mapping_domains_id",
-        name="Mapping - Domains ID",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
-    )
-    MAPPING_CODELIST_TYPE = __SWSDatatable(
-        id="datatables.mapping_codelist_type",
-        name="Mapping Codelist type",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
-    )
-    MAPPING_CODE_CORRECTION = __SWSDatatable(
-        id="datatables.aggregates_mapping_code_correction",
-        name="Mapping - Code correction",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_SDMX_COLUMN_NAMES = __SWSDatatable(
-        id="datatables.mapping_sdmx_col_names",
-        name="Mapping - SDMX column names",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_name STRING, external_name STRING, delete BOOLEAN, add BOOLEAN, default_value STRING",
-    )
-    MAPPING_SDMX_CODES = __SWSDatatable(
-        id="datatables.mapping_pre_dissemination",
-        name="Mapping - Pre dissemination",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_UNITS_OF_MEASURE = __SWSDatatable(
-        id="datatables.mapping_units_of_measure",
-        name="Mapping - Units of measure",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-
-    # Non-SWS Sources Tables
-    FAOSTAT_CODE_MAPPING = __SWSDatatable(
-        id="datatables.faostat_code_mapping",
-        name="FAOSTAT Code Mapping",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, faostat_code_type STRING, faostat_code STRING, mapping_type STRING, mapped_code STRING",
-    )
-    FS_INPUT_MAPPING = __SWSDatatable(
-        id="datatables.fs_input_mapping",
-        name="FS Input Mapping",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, DomainCode STRING, Code STRING, Var2Code STRING, Var3Code STRING, Mult INT, 3YrAvgFlag INT, Flag STRING",
-    )
-    HCES_INPUT_MAPPING = __SWSDatatable(
-        id="datatables.hces_input_mapping",
-        name="HCES Input Mapping",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, variable STRING, indicator STRING, element STRING, decimals STRING",
-    )
-
-
-class DatasetTables:
-    class __SWSTable:
-        def __init__(self, postgres_id: str, iceberg_id: str, schema: str):
-            self.postgres_id = postgres_id
-            self.iceberg_id = iceberg_id
-            self.schema = schema
-
-    def __get_obs_coord_schema(self) -> str:
-
-        obs_coord_schema_prefix = (
-            "id BIGINT, approved_observation BIGINT, num_version BIGINT, "
-        )
-
-        dimensions_schema = (
-            " INT, ".join(self.__dataset_details["dimensionColumns"]) + " INT"
-        )
-
-        return obs_coord_schema_prefix + dimensions_schema
-
-    def __init__(self, dataset_id: str, dataset_details: dict) -> None:
-        self.__dataset_id = dataset_id
-        self.__dataset_details = dataset_details
-
-        # Data
-        self.OBSERVATION = self.__SWSTable(
-            postgres_id=f"{self.__dataset_id}.observation",
-            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_observation",
-            schema="id BIGINT, observation_coordinates BIGINT, version INT, value FLOAT, flag_obs_status STRING, flag_method STRING, created_on TIMESTAMP, created_by INT, replaced_on TIMESTAMP",
-        )
-        self.OBSERVATION_COORDINATE = self.__SWSTable(
-            postgres_id=f"{self.__dataset_id}.observation_coordinate",
-            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_observation_coordinate",
-            schema=self.__get_obs_coord_schema(),
-        )
-        self.METADATA = self.__SWSTable(
-            postgres_id=f"{self.__dataset_id}.metadata",
-            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata",
-            schema="id BIGINT, observation BIGINT, metadata_type INT, language INT, copy_metadata BIGINT",
-        )
-        self.METADATA_ELEMENT = self.__SWSTable(
-            postgres_id=f"{self.__dataset_id}.metadata_element",
-            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata_element",
-            schema="id BIGINT, metadata INT, metadata_element_type INT, value STRING",
-        )
-
-        # Reference data
-        self.CODELISTS = [
-            self.__SWSTable(
-                postgres_id=dimension["codelist"]["table"],
-                iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{dimension['codelist']['table'].split('.')[1]}",
-                schema=CODELIST_SCHEMA,
-            )
-            for dimension in dataset_details["dimensions"]
-        ]
-
-    FLAG_METHOD = __SWSTable(
-        postgres_id="reference_data.flag_method",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.flag_method",
-        schema=FLAGLIST_SCHEMA,
-    )
-    FLAG_OBS_STATUS = __SWSTable(
-        postgres_id="reference_data.flag_obs_status",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.flag_obs_status",
-        schema=FLAGLIST_SCHEMA,
-    )
-    METADATA_TYPE = __SWSTable(
-        postgres_id="reference_data.metadata_type",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_type",
-        schema="id INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN",
-    )
-    METADATA_ELEMENT_TYPE = __SWSTable(
-        postgres_id="reference_data.metadata_element_type",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_element_type",
-        schema="id INT, metadata_type INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN, private BOOLEAN",
-    )
-
-    LANGUAGE = __SWSTable(
-        postgres_id="reference_data.language",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.language",
-        schema="id INT, country_code STRING, description STRING",
-    )
-
-    UNIT_OF_MEASURE = __SWSTable(
-        postgres_id="reference_data.unit_of_measure",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.unit_of_measure",
-        schema="id INT, code STRING, sdmx_code STRING, metric BOOLEAN, description STRING, symbol STRING, base_unit STRING, multiplier DECIMAL",
-    )
-
-    # Operational data
-    USER = __SWSTable(
-        postgres_id="operational_data.user",
-        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.user",
-        schema="id INT, username STRING, preferences INT, email STRING, active BOOLEAN, settings STRING",
-    )
-
-
-class IcebergTable:
-    def __init__(self, level: str, iceberg_id: str, path: str, csv_path: str):
-        self.level = level.lower()
-        self.iceberg_id = iceberg_id
-        self.table = iceberg_id.split(".")[-1]
-        self.path = path
-        self.csv_prefix = csv_path.rsplit("/", 1)[0]
-        self.csv_path = csv_path
-
-
-class IcebergTables:
-
-    def __init__(self, dataset_id: str, tag_name: str, domain: str = "") -> None:
-        self.__dataset_id = dataset_id
-        self.__tag_name = tag_name
-
-        self.BRONZE = self._create_iceberg_table("BRONZE")
-        self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
-
-        # GOLD tables with specific suffixes
-        self.GOLD_SDMX = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sdmx_disseminated"
-        )
-        self.GOLD_SWS_VALIDATED = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sws_validated"
-        )
-        self.GOLD_SWS_DISSEMINATED = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sws_disseminated"
-        )
-
-    def _create_iceberg_table(
-        self, level: str, prefix: str = "", suffix: str = ""
-    ) -> IcebergTable:
-        database = getattr(IcebergDatabases, f"{level}_DATABASE")
-        scheme = getattr(IcebergDatabases, f"{level}_SCHEME")
-
-        if prefix != "":
-            prefix = f"{prefix}_".lower()
-        if suffix != "":
-            suffix = f"_{suffix}".lower()
-
-        iceberg_id = f"{database}.{prefix}{self.__dataset_id}{suffix}"
-        path = f"{scheme}/{prefix}{self.__dataset_id}{suffix}"
-        csv_path = f"{CACHED_CSV_FOLDER}/{scheme}/{prefix}{self.__dataset_id}{suffix}/{self.__tag_name}.csv"
-
-        return IcebergTable(level, iceberg_id, path, csv_path)