sws-spark-dissemination-helper 0.0.141__tar.gz → 0.0.143__tar.gz

{sws_spark_dissemination_helper-0.0.141 → sws_spark_dissemination_helper-0.0.143}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.141
+Version: 0.0.143
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.141 → sws_spark_dissemination_helper-0.0.143}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.141"
+version = "0.0.143"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.141 → sws_spark_dissemination_helper-0.0.143}/src/sws_spark_dissemination_helper/constants.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from pyspark.sql.functions import col, lit
 
 SPARK_POSTGRES_DRIVER = "org.postgresql.Driver"
@@ -34,10 +36,14 @@ class DomainFilters:
 class DatasetDatatables:
 
     class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
+        def __init__(
+            self, id: str, name: str, schema: str, join_columns: List[str] = []
+        ):
             self.id = id
+            self.iceberg_id = f"{IcebergDatabases.BRONZE_DATABASE}{id.split('.')[1]}"
             self.name = name
             self.schema = schema
+            self.join_columns = join_columns
 
     # Aggregation Tables
     AGGREGATES_COMPOSITION = __SWSDatatable(
@@ -50,22 +56,37 @@ class DatasetDatatables:
         name="Aggregation - Aggregates per elements",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
     )
-    
+
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
         name="Dissemination - {type} list",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
     DISSEMINATION_EXCEPTIONS = __SWSDatatable(
         id="datatables.dissemination_exception",
         name="Dissemination - Exceptions",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
+        join_columns=[
+            "domain",
+            " dim1_code",
+            " dim2_code",
+            " dim3_code",
+            " dim4_code",
+            " dim5_code",
+            " dim6_code",
+            " dim7_code",
+            " status_flag",
+            " method_flag",
+        ],
     )
+    # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
         name="Dissemination - Item list - FAOSTAT",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
 
     # Mapping Tables
@@ -73,19 +94,23 @@ class DatasetDatatables:
         id="datatables.aggregates_mapping_domains_id",
         name="Mapping - Domains ID",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
+        join_columns=["domain", "sws_source_id"],
     )
     MAPPING_CODELIST_TYPE = __SWSDatatable(
         id="datatables.mapping_codelist_type",
         name="Mapping Codelist type",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
+        join_columns=["domain", "col_name"],
     )
     MAPPING_CODE_CORRECTION = __SWSDatatable(
         id="datatables.aggregates_mapping_code_correction",
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
+        join_columns=["domain", "old_code", "var_type", "mapping_type"],
     )
 
     # Non-SWS Sources Tables
+    # TODO To deprecate
     FAOSTAT_CODE_MAPPING = __SWSDatatable(
         id="datatables.faostat_code_mapping",
         name="FAOSTAT Code Mapping",
@@ -223,9 +248,7 @@ class IcebergTables:
         self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SWS = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sws"
-        )
+        self.GOLD_SWS = self._create_iceberg_table("GOLD", prefix=domain, suffix="sws")
         self.GOLD_SDMX = self._create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )