sws-spark-dissemination-helper 0.0.140__tar.gz → 0.0.141__tar.gz

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.141}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.140
+Version: 0.0.141
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.141}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.140"
+version = "0.0.141"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.141}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -235,7 +235,7 @@ class SWSEasyIcebergSparkHelper:
         # ----------------
 
         select_statement = """
-        select o.id,
+            o.id,
             o.value,
             u.email,
             o.created_on,
@@ -243,24 +243,33 @@ class SWSEasyIcebergSparkHelper:
             o.version"""
 
         from_statement = f"""
-        from {self.dataset_tables.OBSERVATION.iceberg_id} o
-            join {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
-            left join {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} as oc on oc.id = o.observation_coordinates"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
 
         id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
         for flag_col in self.flag_columns:
-            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} as {flag_col}"
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
 
         id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
         for i, (dim_col, cl) in enumerate(
             zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
         ):
-            select_statement += f",\nd{i}.code as {dim_col}"
-            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
 
-        final_query = select_statement + from_statement
+        final_query = "SELECT " + hint_statement + select_statement + from_statement
         if not self.keep_history:
-            final_query += "\nwhere o.replaced_on is null"
+            final_query += "\nWHERE o.replaced_on IS NULL"
 
         logging.info("Final query for merging observation and observation_coordinares")
         logging.info(final_query)

{sws_spark_dissemination_helper-0.0.140 → sws_spark_dissemination_helper-0.0.141}/src/sws_spark_dissemination_helper/constants.py

@@ -39,6 +39,18 @@ class DatasetDatatables:
             self.name = name
             self.schema = schema
 
+    # Aggregation Tables
+    AGGREGATES_COMPOSITION = __SWSDatatable(
+        id="datatables.aggregates_composition",
+        name="Aggregation - Composition",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, aggregation_type STRING, group_code STRING, child_code STRING, group_name STRING, child_name STRING, link_code STRING, factor STRING",
+    )
+    AGGREGATES_ELEMENTS = __SWSDatatable(
+        id="datatables.aggregates_elements",
+        name="Aggregation - Aggregates per elements",
+        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
+    )
+    
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
@@ -72,21 +84,6 @@ class DatasetDatatables:
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
     )
-    MAPPING_SDMX_COLUMN_NAMES = __SWSDatatable(
-        id="datatables.mapping_sdmx_col_names",
-        name="Mapping - SDMX column names",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_name STRING, external_name STRING, delete BOOLEAN, add BOOLEAN, default_value STRING",
-    )
-    MAPPING_SDMX_CODES = __SWSDatatable(
-        id="datatables.mapping_pre_dissemination",
-        name="Mapping - Pre dissemination",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, internal_code STRING, external_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
-    )
-    MAPPING_UNITS_OF_MEASURE = __SWSDatatable(
-        id="datatables.mapping_units_of_measure",
-        name="Mapping - Units of measure",
-        schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, sws_code STRING, sws_multiplier INT, sdmx_code STRING, sdmx_multiplier INT, value_multiplier INT, delete BOOLEAN, mapping_type STRING",
-    )
 
     # Non-SWS Sources Tables
     FAOSTAT_CODE_MAPPING = __SWSDatatable(