sws-spark-dissemination-helper 0.0.128__tar.gz → 0.0.130__tar.gz

{sws_spark_dissemination_helper-0.0.128 → sws_spark_dissemination_helper-0.0.130}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.128
+Version: 0.0.130
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.128 → sws_spark_dissemination_helper-0.0.130}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.128"
+version = "0.0.130"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.128 → sws_spark_dissemination_helper-0.0.130}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -159,7 +159,7 @@ class SWSEasyIcebergSparkHelper:
             left join reference_data.dim_element_fao d1 on d1.id = oc.dim_element_fao
             left join reference_data.dim_item_cpc d2 on d2.id = oc.dim_item_cpc
             left join reference_data.dim_time_series_years d3 on d3.id = oc.dim_time_series_years
-        where o.replaced_on is null,
+        where o.replaced_on is null
         """
 
         # ----------------
@@ -229,6 +229,45 @@ class SWSEasyIcebergSparkHelper:
 
         return df_obs_denorm
 
+    def _gen_denormalized_observation_sql(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+        select o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version,
+            o.flag_obs_status,
+            o.flag_method
+        """
+
+        from_statement = f"""
+        from {self.dataset_tables.OBSERVATION.iceberg_id} o
+            join {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            left join {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} as oc on oc.id = o.observation_coordinates
+        """
+
+        for i, (dim, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f',\nd{i}.code as "{dim}"'
+            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{dim}"
+
+        final_query = select_statement + from_statement
+        if not self.keep_history:
+            final_query += "where o.replaced_on is null"
+
+        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        return df_obs_denorm
+
     def _gen_denormalized_metadata(self) -> DataFrame:
         """Original query upon which the below computation is based
 
@@ -292,6 +331,32 @@ class SWSEasyIcebergSparkHelper:
 
         return df_meta_denorm
 
+    def _gen_denormalized_metadata_sql(self) -> DataFrame:
+        # ----------------
+        # Generate denormalized observation table
+        # ----------------
+
+        logging.info("meta_denorm start")
+
+        df_meta_denorm = self.spark.sql(
+            f"""
+        select m.observation as observation_id,
+            mt.code as type,
+            met.code as element_type,
+            l.country_code as language,
+            me.value
+        from {self.dataset_tables.METADATA_ELEMENT.iceberg_id} me
+            left join {self.dataset_tables.METADATA.iceberg_id} m on m.id = me.metadata
+            left join {self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id} met on met.id = me.metadata_element_type
+            left join {self.dataset_tables.METADATA_TYPE.iceberg_id} mt on mt.id = m.metadata_type
+            left join {self.dataset_tables.LANGUAGE.iceberg_id} reference_data.language l on l.id = m.language
+        """
+        )
+
+        logging.info("meta_denorm write")
+
+        return df_meta_denorm
+
     def _gen_grouped_metadata(self) -> DataFrame:
         return (
             self._gen_denormalized_metadata()
@@ -312,6 +377,26 @@ class SWSEasyIcebergSparkHelper:
             .agg(F.collect_list("metadata").alias("metadata"))
         )
 
+    def _gen_grouped_metadata_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_metadata_sql()
+            .select(
+                col("observation_id"),
+                F.create_map(
+                    lit("type"),
+                    col("type"),
+                    lit("element_type"),
+                    col("element_type"),
+                    lit("language"),
+                    col("language"),
+                    lit("value"),
+                    col("value"),
+                ).alias("metadata"),
+            )
+            .groupby("observation_id")
+            .agg(F.collect_list("metadata").alias("metadata"))
+        )
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -324,8 +409,23 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self) -> DataFrame:
-        self.df_denorm = self._gen_denormalied_data()
+    def _gen_denormalied_data_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
+    def write_data_to_iceberg_and_csv(self, sql=False) -> DataFrame:
+        if sql:
+            self.df_denorm = self._gen_denormalied_data_sql()
+        else:
+            self.df_denorm = self._gen_denormalied_data()
 
         self.df_denorm.writeTo(self.iceberg_tables.TABLE.iceberg_id).createOrReplace()