sws-spark-dissemination-helper 0.0.140__tar.gz → 0.0.142__tar.gz

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.142}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.140
+Version: 0.0.142
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.142}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.140"
+version = "0.0.142"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.142}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -235,7 +235,7 @@ class SWSEasyIcebergSparkHelper:
         # ----------------
 
         select_statement = """
-        select o.id,
+            o.id,
             o.value,
             u.email,
             o.created_on,
@@ -243,24 +243,33 @@ class SWSEasyIcebergSparkHelper:
             o.version"""
 
         from_statement = f"""
-        from {self.dataset_tables.OBSERVATION.iceberg_id} o
-            join {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
-            left join {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} as oc on oc.id = o.observation_coordinates"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
 
         id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
         for flag_col in self.flag_columns:
-            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} as {flag_col}"
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
 
         id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
         for i, (dim_col, cl) in enumerate(
             zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
         ):
-            select_statement += f",\nd{i}.code as {dim_col}"
-            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
 
-        final_query = select_statement + from_statement
+        final_query = "SELECT " + hint_statement + select_statement + from_statement
         if not self.keep_history:
-            final_query += "\nwhere o.replaced_on is null"
+            final_query += "\nWHERE o.replaced_on IS NULL"
 
         logging.info("Final query for merging observation and observation_coordinares")
         logging.info(final_query)

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.142}/src/sws_spark_dissemination_helper/constants.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from pyspark.sql.functions import col, lit
 
 SPARK_POSTGRES_DRIVER = "org.postgresql.Driver"
@@ -34,26 +36,56 @@ class DomainFilters:
 class DatasetDatatables:
 
     class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
+        def __init__(
+            self, id: str, name: str, schema: str, join_columns: List[str] = []
+        ):
             self.id = id
             self.name = name
             self.schema = schema
+            self.join_columns = join_columns
+
+    # Aggregation Tables
+    AGGREGATES_COMPOSITION = __SWSDatatable(
+        id="datatables.aggregates_composition",
+        name="Aggregation - Composition",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, aggregation_type STRING, group_code STRING, child_code STRING, group_name STRING, child_name STRING, link_code STRING, factor STRING",
+    )
+    AGGREGATES_ELEMENTS = __SWSDatatable(
+        id="datatables.aggregates_elements",
+        name="Aggregation - Aggregates per elements",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
+    )
 
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
         name="Dissemination - {type} list",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
     DISSEMINATION_EXCEPTIONS = __SWSDatatable(
         id="datatables.dissemination_exception",
         name="Dissemination - Exceptions",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
+        join_columns=[
+            "domain",
+            " dim1_code",
+            " dim2_code",
+            " dim3_code",
+            " dim4_code",
+            " dim5_code",
+            " dim6_code",
+            " dim7_code",
+            " status_flag",
+            " method_flag",
+        ],
     )
+    # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
         name="Dissemination - Item list - FAOSTAT",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
 
     # Mapping Tables
@@ -61,34 +93,23 @@ class DatasetDatatables:
         id="datatables.aggregates_mapping_domains_id",
         name="Mapping - Domains ID",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
+        join_columns=["domain", "sws_source_id"],
     )
     MAPPING_CODELIST_TYPE = __SWSDatatable(
         id="datatables.mapping_codelist_type",
         name="Mapping Codelist type",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
+        join_columns=["domain", "col_name"],
     )
     MAPPING_CODE_CORRECTION = __SWSDatatable(
         id="datatables.aggregates_mapping_code_correction",
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_SDMX_COLUMN_NAMES = __SWSDatatable(
-        id="datatables.mapping_sdmx_col_names",
-        name="Mapping - SDMX column names",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_name STRING, external_name STRING, delete BOOLEAN, add BOOLEAN, default_value STRING",
-    )
-    MAPPING_SDMX_CODES = __SWSDatatable(
-        id="datatables.mapping_pre_dissemination",
-        name="Mapping - Pre dissemination",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_UNITS_OF_MEASURE = __SWSDatatable(
-        id="datatables.mapping_units_of_measure",
-        name="Mapping - Units of measure",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, sws_code STRING, sws_multiplier INT, sdmx_code STRING, sdmx_multiplier INT, value_multiplier INT, delete BOOLEAN, mapping_type STRING",
+        join_columns=["domain", "old_code", "var_type", "mapping_type"],
     )
 
     # Non-SWS Sources Tables
+    # TODO To deprecate
     FAOSTAT_CODE_MAPPING = __SWSDatatable(
         id="datatables.faostat_code_mapping",
         name="FAOSTAT Code Mapping",
@@ -226,9 +247,7 @@ class IcebergTables:
         self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SWS = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sws"
-        )
+        self.GOLD_SWS = self._create_iceberg_table("GOLD", prefix=domain, suffix="sws")
         self.GOLD_SDMX = self._create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )