sws-spark-dissemination-helper 0.0.159__tar.gz → 0.0.161__tar.gz

{sws_spark_dissemination_helper-0.0.159 → sws_spark_dissemination_helper-0.0.161}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.159
+Version: 0.0.161
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.159 → sws_spark_dissemination_helper-0.0.161}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.159"
+version = "0.0.161"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.159 → sws_spark_dissemination_helper-0.0.161}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -276,11 +276,17 @@ class SWSEasyIcebergSparkHelper:
         if not self.keep_history:
             final_query += "\nWHERE o.replaced_on IS NULL"
 
-        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info("Final query for merging observation and observation_coordinates")
         logging.info(final_query)
 
         df_obs_denorm = self.spark.sql(final_query)
 
+        df_obs_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_OBSERVATION.table} write")
+
         return df_obs_denorm
 
     def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
@@ -418,7 +424,13 @@ class SWSEasyIcebergSparkHelper:
 
         df_meta_denorm = self.spark.sql(
             f"""
-        select m.observation as observation_id,
+        select 
+            /*+
+                BROADCAST({self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.METADATA_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.LANGUAGE.iceberg_id}),
+            */
+            m.observation as observation_id,
             mt.code as type,
             met.code as element_type,
             l.country_code as language,
@@ -431,7 +443,11 @@ class SWSEasyIcebergSparkHelper:
         """
         )
 
-        logging.info("meta_denorm write")
+        df_meta_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_METADATA.table} write")
 
         return df_meta_denorm
 
@@ -456,25 +472,31 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_grouped_metadata_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_metadata_sql()
-            .select(
-                col("observation_id"),
-                F.create_map(
-                    lit("type"),
-                    col("type"),
-                    lit("element_type"),
-                    col("element_type"),
-                    lit("language"),
-                    col("language"),
-                    lit("value"),
-                    col("value"),
-                ).alias("metadata"),
-            )
-            .groupby("observation_id")
-            .agg(F.collect_list("metadata").alias("metadata"))
+        df_meta_grouped = self.spark.sql(
+            f"""
+        SELECT
+            observation_id,
+            collect_list(
+                map(
+                    'type', type,
+                    'element_type', element_type,
+                    'language', language,
+                    'value', value
+                )
+            ) AS metadata
+        FROM {self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id}
+        GROUP BY observation_id
+        """
         )
 
+        df_meta_grouped.writeTo(
+            self.iceberg_tables.GROUPED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.GROUPED_METADATA.table} write")
+
+        return df_meta_grouped
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -488,15 +510,15 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_denormalied_data_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_observation_sql()
-            .alias("o")
-            .join(
-                self._gen_grouped_metadata_sql().alias("m"),
-                col("o.id") == col("m.observation_id"),
-                "left",
-            )
-            .drop("m.observation_id")
+        return self.spark.sql(
+            f"""
+            SELECT
+                o.*,
+                m.metadata
+            FROM {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS o
+            LEFT JOIN {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS m
+                ON o.id = m.observation_id
+            """
         )
 
     def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
@@ -669,3 +691,29 @@ class SWSEasyIcebergSparkHelper:
             logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")
+
+
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1

{sws_spark_dissemination_helper-0.0.159 → sws_spark_dissemination_helper-0.0.161}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -496,8 +496,8 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
         new_diss_table = BaseDisseminatedTagTable(
-            id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold pre-SDMX csv",
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
             description="Gold table containing the tag data without any processing cached in csv",
             layer=TableLayer.GOLD,
             private=True,
@@ -515,3 +515,52 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faosta_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df

{sws_spark_dissemination_helper-0.0.159 → sws_spark_dissemination_helper-0.0.161}/src/sws_spark_dissemination_helper/constants.py

@@ -254,6 +254,9 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
+        self.DENORMALIZED_OBSERVATION = self._create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self._create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self._create_iceberg_table("BRONZE", suffix="grouped_metadata")
         self.TABLE = self._create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self._create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self._create_iceberg_table("BRONZE")
@@ -274,6 +277,12 @@ class IcebergTables:
         self.GOLD_PRE_SDMX = self._create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
+        self.GOLD_FAOSTAT = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
     def _create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""