sws-spark-dissemination-helper 0.0.159__py3-none-any.whl → 0.0.167__py3-none-any.whl

sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -276,11 +276,17 @@ class SWSEasyIcebergSparkHelper:
         if not self.keep_history:
             final_query += "\nWHERE o.replaced_on IS NULL"
 
-        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info("Final query for merging observation and observation_coordinates")
         logging.info(final_query)
 
         df_obs_denorm = self.spark.sql(final_query)
 
+        df_obs_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_OBSERVATION.table} write")
+
         return df_obs_denorm
 
     def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
@@ -418,7 +424,13 @@ class SWSEasyIcebergSparkHelper:
 
         df_meta_denorm = self.spark.sql(
             f"""
-        select m.observation as observation_id,
+        select 
+            /*+
+                BROADCAST({self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.METADATA_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.LANGUAGE.iceberg_id})
+            */
+            m.observation as observation_id,
             mt.code as type,
             met.code as element_type,
             l.country_code as language,
@@ -431,7 +443,11 @@ class SWSEasyIcebergSparkHelper:
         """
         )
 
-        logging.info("meta_denorm write")
+        df_meta_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_METADATA.table} write")
 
         return df_meta_denorm
 
@@ -456,25 +472,31 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_grouped_metadata_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_metadata_sql()
-            .select(
-                col("observation_id"),
-                F.create_map(
-                    lit("type"),
-                    col("type"),
-                    lit("element_type"),
-                    col("element_type"),
-                    lit("language"),
-                    col("language"),
-                    lit("value"),
-                    col("value"),
-                ).alias("metadata"),
-            )
-            .groupby("observation_id")
-            .agg(F.collect_list("metadata").alias("metadata"))
+        df_meta_grouped = self.spark.sql(
+            f"""
+        SELECT
+            observation_id,
+            collect_list(
+                map(
+                    'type', type,
+                    'element_type', element_type,
+                    'language', language,
+                    'value', value
+                )
+            ) AS metadata
+        FROM {self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id}
+        GROUP BY observation_id
+        """
         )
 
+        df_meta_grouped.writeTo(
+            self.iceberg_tables.GROUPED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.GROUPED_METADATA.table} write")
+
+        return df_meta_grouped
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -488,15 +510,19 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_denormalied_data_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_observation_sql()
-            .alias("o")
-            .join(
-                self._gen_grouped_metadata_sql().alias("m"),
-                col("o.id") == col("m.observation_id"),
-                "left",
-            )
-            .drop("m.observation_id")
+        self._gen_denormalized_observation_sql()
+        self._gen_denormalized_metadata_sql()
+        self._gen_grouped_metadata_sql()
+
+        return self.spark.sql(
+            f"""
+            SELECT
+                o.*,
+                m.metadata
+            FROM {self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id} AS o
+            LEFT JOIN {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS m
+                ON o.id = m.observation_id
+            """
         )
 
     def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
@@ -669,3 +695,29 @@ class SWSEasyIcebergSparkHelper:
             logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")
+
+
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -271,6 +271,60 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def write_gold_faostat_unfiltered_data_to_iceberg_and_csv(
+        self, df: DataFrame
+    ) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(
+            self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT unfiltered table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT unfiltered tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:
@@ -496,8 +550,8 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
         new_diss_table = BaseDisseminatedTagTable(
-            id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold pre-SDMX csv",
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
             description="Gold table containing the tag data without any processing cached in csv",
             layer=TableLayer.GOLD,
             private=True,
@@ -515,3 +569,101 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_unfiltered_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered Iceberg",
+            description="Gold table containing all the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_csv",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df

sws_spark_dissemination_helper/constants.py

@@ -254,6 +254,9 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
+        self.DENORMALIZED_OBSERVATION = self._create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self._create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self._create_iceberg_table("BRONZE", suffix="grouped_metadata")
         self.TABLE = self._create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self._create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self._create_iceberg_table("BRONZE")
@@ -274,6 +277,12 @@ class IcebergTables:
         self.GOLD_PRE_SDMX = self._create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
+        self.GOLD_FAOSTAT = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
     def _create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""

{sws_spark_dissemination_helper-0.0.159.dist-info → sws_spark_dissemination_helper-0.0.167.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.159
+Version: 0.0.167
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.159.dist-info → sws_spark_dissemination_helper-0.0.167.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=ocuau0WtpyRwui0qwdQ_Rxh4nYPOyZoHpGKaWRa6B3Q,28868
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=u1fFUagXvVPJirSIesuXrCbFMuZ2jQnbz1yjk1MqRwg,25061
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=KZy6v4V3ugzKq_0L8JLmTPClN0hx-9uWpAwNFcs37Og,19339
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=0dxbVkrhdaASapEffF5PFcgKwAMyJoWBxzgymjZ4JyY,25049
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
-sws_spark_dissemination_helper/constants.py,sha256=_VZXdX5ARnlNzqQEJoFqY_g5r5KyF3cez7SZKV6bYrs,12915
+sws_spark_dissemination_helper/constants.py,sha256=zviO6huxWTWonHv4v2M8zKr7HXCDMBGqjHx-eTfGT2A,13487
 sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
-sws_spark_dissemination_helper-0.0.159.dist-info/METADATA,sha256=JMSC-t_4LKJZNufyFNAe9rShRBSlQUHge-06z4slJ4c,2824
-sws_spark_dissemination_helper-0.0.159.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.159.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.159.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.167.dist-info/METADATA,sha256=h27GpuoB4elORRQybwqiKoKkF59JOPk6HzkQ4uDsSjo,2824
+sws_spark_dissemination_helper-0.0.167.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.167.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.167.dist-info/RECORD,,