sws-spark-dissemination-helper 0.0.127__tar.gz → 0.0.129__tar.gz

{sws_spark_dissemination_helper-0.0.127 → sws_spark_dissemination_helper-0.0.129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.127
+Version: 0.0.129
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.127 → sws_spark_dissemination_helper-0.0.129}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.127"
+version = "0.0.129"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.127 → sws_spark_dissemination_helper-0.0.129}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -25,6 +25,7 @@ class SWSEasyIcebergSparkHelper:
         dataset_details: dict = None,
         dataset_tables: DatasetTables = None,
         keep_history: bool = False,
+        write_csv: bool = True,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -35,6 +36,7 @@ class SWSEasyIcebergSparkHelper:
         self.dataset_tables: DatasetTables = dataset_tables
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.keep_history: bool = keep_history
+        self.write_csv: bool = write_csv
 
         if dataset_details is not None:
             (
@@ -157,7 +159,7 @@ class SWSEasyIcebergSparkHelper:
             left join reference_data.dim_element_fao d1 on d1.id = oc.dim_element_fao
             left join reference_data.dim_item_cpc d2 on d2.id = oc.dim_item_cpc
             left join reference_data.dim_time_series_years d3 on d3.id = oc.dim_time_series_years
-        where o.replaced_on is null,
+        where o.replaced_on is null
         """
 
         # ----------------
@@ -227,6 +229,45 @@ class SWSEasyIcebergSparkHelper:
 
         return df_obs_denorm
 
+    def _gen_denormalized_observation_sql(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+        select o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version,
+            o.flag_obs_status,
+            o.flag_method
+        """
+
+        from_statement = f"""
+        from {self.dataset_tables.OBSERVATION.iceberg_id} o
+            join {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            left join {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} as oc on oc.id = o.observation_coordinates
+        """
+
+        for i, (dim, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f",\nd{i}.code as '{dim}'"
+            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{dim}"
+
+        final_query = select_statement + from_statement
+        if not self.keep_history:
+            final_query += "where o.replaced_on is null"
+
+        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        return df_obs_denorm
+
     def _gen_denormalized_metadata(self) -> DataFrame:
         """Original query upon which the below computation is based
 
@@ -290,6 +331,32 @@ class SWSEasyIcebergSparkHelper:
 
         return df_meta_denorm
 
+    def _gen_denormalized_metadata_sql(self) -> DataFrame:
+        # ----------------
+        # Generate denormalized observation table
+        # ----------------
+
+        logging.info("meta_denorm start")
+
+        df_meta_denorm = self.spark.sql(
+            f"""
+        select m.observation as observation_id,
+            mt.code as type,
+            met.code as element_type,
+            l.country_code as language,
+            me.value
+        from {self.dataset_tables.METADATA_ELEMENT.iceberg_id} me
+            left join {self.dataset_tables.METADATA.iceberg_id} m on m.id = me.metadata
+            left join {self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id} met on met.id = me.metadata_element_type
+            left join {self.dataset_tables.METADATA_TYPE.iceberg_id} mt on mt.id = m.metadata_type
+            left join {self.dataset_tables.LANGUAGE.iceberg_id} reference_data.language l on l.id = m.language
+        """
+        )
+
+        logging.info("meta_denorm write")
+
+        return df_meta_denorm
+
     def _gen_grouped_metadata(self) -> DataFrame:
         return (
             self._gen_denormalized_metadata()
@@ -310,6 +377,26 @@ class SWSEasyIcebergSparkHelper:
             .agg(F.collect_list("metadata").alias("metadata"))
         )
 
+    def _gen_grouped_metadata_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_metadata_sql()
+            .select(
+                col("observation_id"),
+                F.create_map(
+                    lit("type"),
+                    col("type"),
+                    lit("element_type"),
+                    col("element_type"),
+                    lit("language"),
+                    col("language"),
+                    lit("value"),
+                    col("value"),
+                ).alias("metadata"),
+            )
+            .groupby("observation_id")
+            .agg(F.collect_list("metadata").alias("metadata"))
+        )
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -322,8 +409,23 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self) -> DataFrame:
-        self.df_denorm = self._gen_denormalied_data()
+    def _gen_denormalied_data_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
+    def write_data_to_iceberg_and_csv(self, sql=False) -> DataFrame:
+        if sql:
+            self.df_denorm = self._gen_denormalied_data_sql()
+        else:
+            self.df_denorm = self._gen_denormalied_data()
 
         self.df_denorm.writeTo(self.iceberg_tables.TABLE.iceberg_id).createOrReplace()
 
@@ -335,16 +437,16 @@ class SWSEasyIcebergSparkHelper:
 
         logging.info(f"Iceberg tag '{self.tag_name}' created")
 
-        df_denorm = self.df_denorm.withColumn(
-            "metadata", F.to_json(col("metadata"))
-        ).coalesce(1)
+        df_denorm = self.df_denorm.withColumn("metadata", F.to_json(col("metadata")))
+        if self.write_csv:
+            df_denorm = df_denorm.coalesce(1)
 
-        save_cache_csv(
-            df=df_denorm,
-            bucket=self.bucket,
-            prefix=self.iceberg_tables.TABLE.csv_prefix,
-            tag_name=self.tag_name,
-        )
+            save_cache_csv(
+                df=df_denorm,
+                bucket=self.bucket,
+                prefix=self.iceberg_tables.TABLE.csv_prefix,
+                tag_name=self.tag_name,
+            )
 
         return df_denorm
 
@@ -372,21 +474,22 @@ class SWSEasyIcebergSparkHelper:
         )
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
-        new_csv_table = BaseDisseminatedTagTable(
-            id="unfiltered_csv",
-            name="Unfiltered csv",
-            description="Csv table containing all the raw data imported from the SWS and denormalized",
-            layer=TableLayer.CACHE,
-            private=True,
-            debug=True,
-            type=TableType.CSV,
-            path=self.iceberg_tables.TABLE.csv_path,
-            structure={"columns": self.df_denorm.schema.jsonValue()["fields"]},
-        )
-        tag = tags.add_dissemination_table(
-            self.dataset_id, self.tag_name, new_csv_table
-        )
-        logging.debug(f"Tag with Added csv Table: {tag}")
+        if self.write_csv:
+            new_csv_table = BaseDisseminatedTagTable(
+                id="unfiltered_csv",
+                name="Unfiltered csv",
+                description="Csv table containing all the raw data imported from the SWS and denormalized",
+                layer=TableLayer.CACHE,
+                private=True,
+                debug=True,
+                type=TableType.CSV,
+                path=self.iceberg_tables.TABLE.csv_path,
+                structure={"columns": self.df_denorm.schema.jsonValue()["fields"]},
+            )
+            tag = tags.add_dissemination_table(
+                self.dataset_id, self.tag_name, new_csv_table
+            )
+            logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Unfiltered data tags successfully written")
 
@@ -418,15 +521,18 @@ class SWSEasyIcebergSparkHelper:
 
         disseminated_tag_df = self.filtered_df.withColumn(
             "metadata", F.to_json(col("metadata"))
-        ).coalesce(1)
-
-        save_cache_csv(
-            df=disseminated_tag_df,
-            bucket=self.bucket,
-            prefix=f"{self.iceberg_tables.TABLE_FILTERED.csv_prefix}",
-            tag_name=self.tag_name,
         )
 
+        if self.write_csv:
+            disseminated_tag_df = disseminated_tag_df.coalesce(1)
+
+            save_cache_csv(
+                df=disseminated_tag_df,
+                bucket=self.bucket,
+                prefix=f"{self.iceberg_tables.TABLE_FILTERED.csv_prefix}",
+                tag_name=self.tag_name,
+            )
+
         return disseminated_tag_df
 
     def write_sws_filtered_dissemination_tag(self, tags: Tags):
@@ -452,20 +558,21 @@ class SWSEasyIcebergSparkHelper:
         )
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
-        new_csv_table = BaseDisseminatedTagTable(
-            id="filtered_csv",
-            name="Filtered csv",
-            description="Csv table containing the raw data imported from the SWS, denormalized and filtered per dimension cached",
-            layer=TableLayer.CACHE,
-            private=True,
-            type=TableType.CSV,
-            path=self.iceberg_tables.TABLE_FILTERED.csv_path,
-            structure={"columns": self.filtered_df.schema.jsonValue()["fields"]},
-        )
-        tag = tags.add_dissemination_table(
-            self.dataset_id, self.tag_name, new_csv_table
-        )
+        if self.write_csv:
+            new_csv_table = BaseDisseminatedTagTable(
+                id="filtered_csv",
+                name="Filtered csv",
+                description="Csv table containing the raw data imported from the SWS, denormalized and filtered per dimension cached",
+                layer=TableLayer.CACHE,
+                private=True,
+                type=TableType.CSV,
+                path=self.iceberg_tables.TABLE_FILTERED.csv_path,
+                structure={"columns": self.filtered_df.schema.jsonValue()["fields"]},
+            )
+            tag = tags.add_dissemination_table(
+                self.dataset_id, self.tag_name, new_csv_table
+            )
 
-        logging.debug(f"Tag with Added csv Table: {tag}")
+            logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")