sws-spark-dissemination-helper 0.0.79__py3-none-any.whl → 0.0.183__py3-none-any.whl

sws_spark_dissemination_helper/constants.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from pyspark.sql.functions import col, lit
 
 SPARK_POSTGRES_DRIVER = "org.postgresql.Driver"
@@ -34,26 +36,70 @@ class DomainFilters:
 class DatasetDatatables:
 
     class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
+        def __init__(
+            self, id: str, name: str, schema: str, join_columns: List[str] = []
+        ):
             self.id = id
+            self.iceberg_id = f"{IcebergDatabases.BRONZE_DATABASE}.{id.split('.')[1]}"
             self.name = name
             self.schema = schema
+            self.join_columns = join_columns
+
+    # Aggregation Tables
+    AGGREGATES_COMPOSITION = __SWSDatatable(
+        id="datatables.aggregates_composition",
+        name="Aggregation - Composition",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, aggregation_type STRING, group_code STRING, child_code STRING, group_name STRING, child_name STRING, link_code STRING, factor STRING",
+    )
+    AGGREGATES_ELEMENTS = __SWSDatatable(
+        id="datatables.aggregates_elements",
+        name="Aggregation - Aggregates per elements",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
+    )
 
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
         name="Dissemination - {type} list",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
     DISSEMINATION_EXCEPTIONS = __SWSDatatable(
         id="datatables.dissemination_exception",
         name="Dissemination - Exceptions",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
+        join_columns=[
+            "domain",
+            " dim1_code",
+            " dim2_code",
+            " dim3_code",
+            " dim4_code",
+            " dim5_code",
+            " dim6_code",
+            " dim7_code",
+            " status_flag",
+            " method_flag",
+        ],
+    )
+    DISPLAY_DECIMALS = __SWSDatatable(
+        id="datatables.display_decimals",
+        name="Dissemination - Display Decimals",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, column_1_name STRING, column_1_value STRING, column_2_name STRING, column_2_value STRING, display_decimals STRING",
+        join_columns=[
+            "domain",
+            "column_1_name",
+            "column_1_value",
+            "column_2_name",
+            "column_2_value",
+            "display_decimals",
+        ],
     )
+    # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
         name="Dissemination - Item list - FAOSTAT",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
 
     # Mapping Tables
@@ -61,34 +107,23 @@ class DatasetDatatables:
         id="datatables.aggregates_mapping_domains_id",
         name="Mapping - Domains ID",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
+        join_columns=["domain", "sws_source_id"],
     )
     MAPPING_CODELIST_TYPE = __SWSDatatable(
         id="datatables.mapping_codelist_type",
         name="Mapping Codelist type",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
+        join_columns=["domain", "col_name"],
     )
     MAPPING_CODE_CORRECTION = __SWSDatatable(
         id="datatables.aggregates_mapping_code_correction",
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_SDMX_COLUMN_NAMES = __SWSDatatable(
-        id="datatables.mapping_sdmx_col_names",
-        name="Mapping - SDMX column names",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_name STRING, external_name STRING, delete BOOLEAN, add BOOLEAN, default_value STRING",
-    )
-    MAPPING_SDMX_CODES = __SWSDatatable(
-        id="datatables.mapping_pre_dissemination",
-        name="Mapping - Pre dissemination",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_UNITS_OF_MEASURE = __SWSDatatable(
-        id="datatables.mapping_units_of_measure",
-        name="Mapping - Units of measure",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, sws_code STRING, sws_multiplier INT, sdmx_code STRING, sdmx_multiplier INT, value_multiplier INT, delete BOOLEAN, mapping_type STRING",
+        join_columns=["domain", "old_code", "var_type", "mapping_type"],
     )
 
     # Non-SWS Sources Tables
+    # TODO To deprecate
     FAOSTAT_CODE_MAPPING = __SWSDatatable(
         id="datatables.faostat_code_mapping",
         name="FAOSTAT Code Mapping",
@@ -150,6 +185,11 @@ class DatasetTables:
             iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata_element",
             schema="id BIGINT, metadata INT, metadata_element_type INT, value STRING",
         )
+        self.TAG_OBSERVATION = self.__SWSTable(
+            postgres_id=f"{self.__dataset_id}.tag_observation",
+            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_tag_observation",
+            schema="tag BIGINT, observation INT",
+        )
 
         # Reference data
         self.CODELISTS = [
@@ -181,18 +221,21 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_element_type",
         schema="id INT, metadata_type INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN, private BOOLEAN",
     )
-
     LANGUAGE = __SWSTable(
         postgres_id="reference_data.language",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.language",
         schema="id INT, country_code STRING, description STRING",
     )
-
     UNIT_OF_MEASURE = __SWSTable(
         postgres_id="reference_data.unit_of_measure",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.unit_of_measure",
         schema="id INT, code STRING, sdmx_code STRING, metric BOOLEAN, description STRING, symbol STRING, base_unit STRING, multiplier DECIMAL",
     )
+    DATASET = __SWSTable(
+        postgres_id="reference_data.dataset",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.dataset",
+        schema="id INT, xml_name STRING",
+    )
 
     # Operational data
     USER = __SWSTable(
@@ -200,6 +243,11 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.user",
         schema="id INT, username STRING, preferences INT, email STRING, active BOOLEAN, settings STRING",
     )
+    TAG = __SWSTable(
+        postgres_id="operational_data.tag",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.tag",
+        schema="id INT, name STRING, reference_date DATE, dataset INT, type STRING, released_ON DATE, released_by INT, properties STRING",
+    )
 
 
 class IcebergTable:
@@ -218,24 +266,44 @@ class IcebergTables:
         self.__dataset_id = dataset_id
         self.__tag_name = tag_name
 
-        self.BRONZE = self._create_iceberg_table("BRONZE")
-        self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
+        # TODO Fix later with a more appropriate DATABASE
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_observation"
+        )
+        self.DENORMALIZED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="denormalized_metadata"
+        )
+        self.GROUPED_METADATA = self.create_iceberg_table(
+            "BRONZE", suffix="grouped_metadata"
+        )
+        self.TABLE = self.create_iceberg_table("BRONZE")
+        self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
+        self.BRONZE = self.create_iceberg_table("BRONZE")
+        self.BRONZE_DISS_TAG = self.create_iceberg_table("BRONZE", suffix="diss_tag")
+        self.SILVER = self.create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SDMX = self._create_iceberg_table(
+        self.GOLD_SWS = self.create_iceberg_table("GOLD", prefix=domain, suffix="sws")
+        self.GOLD_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )
-        self.GOLD_SWS_VALIDATED = self._create_iceberg_table(
+        self.GOLD_SWS_VALIDATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_validated"
         )
-        self.GOLD_SWS_DISSEMINATED = self._create_iceberg_table(
+        self.GOLD_SWS_DISSEMINATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_disseminated"
         )
-        self.GOLD_PRE_SDMX = self._create_iceberg_table(
+        self.GOLD_PRE_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
+        self.GOLD_FAOSTAT = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
-    def _create_iceberg_table(
+    def create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""
     ) -> IcebergTable:
         database = getattr(IcebergDatabases, f"{level}_DATABASE")

sws_spark_dissemination_helper/utils.py

@@ -363,16 +363,34 @@ def map_codes_and_remove_null_duplicates(
             "diss_flag", F.when(col("delete"), lit(False)).otherwise(col("diss_flag"))
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("delete"),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The observation is not disseminated according to the Mapping - Code correction table"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
+        )
+        # Add mapping message to notes
+        .withColumn(
+            "diss_note",
+            F.when(
+                ~col("is_duplicate")
+                & col("new_dim_code").isNotNull()
+                & (col("new_dim_code") != lit("")),
+                F.array_append(
+                    col("diss_note"),
+                    F.concat(
+                        lit(f"Dimension {col_name} code was changed from "),
+                        col(col_name),
+                        lit(" to "),
+                        col("new_dim_code"),
+                    ),
+                ),
+            ).otherwise(col("diss_note")),
         )
         .withColumn(
             col_name,
@@ -391,18 +409,18 @@ def map_codes_and_remove_null_duplicates(
             ).otherwise(col("diss_flag")),
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("is_duplicate")
                 & col("new_dim_code").isNotNull()
                 & (col("new_dim_code") != lit("")),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The code correction was not applied to avoid observation duplications"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         # Check the domain specific multiplier first and then the standard multiplier
         .withColumn("value", col("value") * F.coalesce(col("multiplier"), lit(1)))

{sws_spark_dissemination_helper-0.0.79.dist-info → sws_spark_dissemination_helper-0.0.183.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.79
+Version: 0.0.183
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
-Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
+Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
 License: MIT License
         
@@ -29,26 +29,30 @@ License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Requires-Dist: annotated-types==0.7.0
-Requires-Dist: boto3==1.34.147
-Requires-Dist: botocore==1.34.147
-Requires-Dist: certifi==2024.7.4
-Requires-Dist: charset-normalizer==3.3.2
-Requires-Dist: idna==3.7
+Requires-Dist: boto3>=1.40.0
+Requires-Dist: botocore>=1.40.0
+Requires-Dist: certifi==2025.1.31
+Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: idna>=3.10
 Requires-Dist: jmespath==1.0.1
+Requires-Dist: numpy==2.0.2
+Requires-Dist: pandas==2.3.3
 Requires-Dist: py4j==0.10.9.7
-Requires-Dist: pydantic-core==2.20.1
-Requires-Dist: pydantic==2.8.2
-Requires-Dist: pyspark==3.5.1
+Requires-Dist: pydantic-core==2.27.2
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: pyspark==3.5.4
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-dotenv==0.19.2
+Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
-Requires-Dist: s3transfer==0.10.2
-Requires-Dist: six==1.16.0
-Requires-Dist: sws-api-client==1.0.7b0
-Requires-Dist: typing-extensions==4.12.2
-Requires-Dist: urllib3==1.26.19
+Requires-Dist: s3transfer>=0.11.2
+Requires-Dist: six==1.17.0
+Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: tzdata==2025.2
+Requires-Dist: urllib3==1.26.20
 Description-Content-Type: text/markdown
 
 # Upload a new version

sws_spark_dissemination_helper-0.0.183.dist-info/RECORD

@@ -0,0 +1,13 @@
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
+sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=atQFiY5Mmo-rzHY7WVWg-Guvg8i1ZcaaoKE4ymTaKdE,27750
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=V_rH4UYoFZfMUc82U-KxeL_o8F44HnMHfLLXoyNxHxs,20016
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=3l5zkEWksnEC-R4mJi8JEHL3ylCMbkMD9a0qbdZQU5E,26345
+sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
+sws_spark_dissemination_helper/constants.py,sha256=cVjTS3xbJNKz-1i7c1dJk2PcOZzQhvuHUp9i4PNIPh4,14055
+sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
+sws_spark_dissemination_helper-0.0.183.dist-info/METADATA,sha256=LDVmzDL6ZDhGrRBd3flpX0TPEIJONpdZJodUGrAvemw,2822
+sws_spark_dissemination_helper-0.0.183.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.183.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.183.dist-info/RECORD,,

sws_spark_dissemination_helper-0.0.79.dist-info/RECORD

@@ -1,11 +0,0 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=tyC3e2LNBes9J2UFR-j7bDlvEffeI0YsiYlMvk0wPxA,16382
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=5BI9uOmYo9E1Q4JdDa4rlQ1LBaNTpzfbwUZjTY1VsYQ,29251
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=wXSz4-SbIcfVfDsN5gsbg6ul5GvVoX59VkfjAmTCToo,14935
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=F0g4N95QIApVNvPFWuQfHphGE320LKoimBRisln7Luk,22033
-sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
-sws_spark_dissemination_helper/constants.py,sha256=hpHHlbojShMWRfyIelXz6c5BqFzO48Oap1zmztlMMrs,11349
-sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.79.dist-info/METADATA,sha256=EXaooj8Ss9G9EfvKets1e7bUc0fTVbc-a9X4zjE26FI,2708
-sws_spark_dissemination_helper-0.0.79.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.79.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.79.dist-info/RECORD,,