sws-spark-dissemination-helper 0.0.160__py3-none-any.whl → 0.0.162__py3-none-any.whl

sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -276,11 +276,17 @@ class SWSEasyIcebergSparkHelper:
         if not self.keep_history:
             final_query += "\nWHERE o.replaced_on IS NULL"
 
-        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info("Final query for merging observation and observation_coordinates")
         logging.info(final_query)
 
         df_obs_denorm = self.spark.sql(final_query)
 
+        df_obs_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_OBSERVATION.table} write")
+
         return df_obs_denorm
 
     def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
@@ -418,7 +424,13 @@ class SWSEasyIcebergSparkHelper:
 
         df_meta_denorm = self.spark.sql(
             f"""
-        select m.observation as observation_id,
+        select 
+            /*+
+                BROADCAST({self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.METADATA_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.LANGUAGE.iceberg_id}),
+            */
+            m.observation as observation_id,
             mt.code as type,
             met.code as element_type,
             l.country_code as language,
@@ -431,7 +443,11 @@ class SWSEasyIcebergSparkHelper:
         """
         )
 
-        logging.info("meta_denorm write")
+        df_meta_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_METADATA.table} write")
 
         return df_meta_denorm
 
@@ -456,25 +472,31 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_grouped_metadata_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_metadata_sql()
-            .select(
-                col("observation_id"),
-                F.create_map(
-                    lit("type"),
-                    col("type"),
-                    lit("element_type"),
-                    col("element_type"),
-                    lit("language"),
-                    col("language"),
-                    lit("value"),
-                    col("value"),
-                ).alias("metadata"),
-            )
-            .groupby("observation_id")
-            .agg(F.collect_list("metadata").alias("metadata"))
+        df_meta_grouped = self.spark.sql(
+            f"""
+        SELECT
+            observation_id,
+            collect_list(
+                map(
+                    'type', type,
+                    'element_type', element_type,
+                    'language', language,
+                    'value', value
+                )
+            ) AS metadata
+        FROM {self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id}
+        GROUP BY observation_id
+        """
         )
 
+        df_meta_grouped.writeTo(
+            self.iceberg_tables.GROUPED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.GROUPED_METADATA.table} write")
+
+        return df_meta_grouped
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -488,15 +510,15 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_denormalied_data_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_observation_sql()
-            .alias("o")
-            .join(
-                self._gen_grouped_metadata_sql().alias("m"),
-                col("o.id") == col("m.observation_id"),
-                "left",
-            )
-            .drop("m.observation_id")
+        return self.spark.sql(
+            f"""
+            SELECT
+                o.*,
+                m.metadata
+            FROM {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS o
+            LEFT JOIN {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS m
+                ON o.id = m.observation_id
+            """
         )
 
     def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
@@ -669,3 +691,29 @@ class SWSEasyIcebergSparkHelper:
             logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")
+
+
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -271,6 +271,31 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:

sws_spark_dissemination_helper/constants.py

@@ -254,6 +254,9 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
+        self.DENORMALIZED_OBSERVATION = self._create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self._create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self._create_iceberg_table("BRONZE", suffix="grouped_metadata")
         self.TABLE = self._create_iceberg_table("BRONZE")
         self.TABLE_FILTERED = self._create_iceberg_table("BRONZE", suffix="filtered")
         self.BRONZE = self._create_iceberg_table("BRONZE")

{sws_spark_dissemination_helper-0.0.160.dist-info → sws_spark_dissemination_helper-0.0.162.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.160
+Version: 0.0.162
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.160.dist-info → sws_spark_dissemination_helper-0.0.162.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=ocuau0WtpyRwui0qwdQ_Rxh4nYPOyZoHpGKaWRa6B3Q,28868
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=u1fFUagXvVPJirSIesuXrCbFMuZ2jQnbz1yjk1MqRwg,25061
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=RcpU8znoyNOfJMLFwrgi7S-BKkzgqh25o2wPLNIxLYc,21246
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=Ko_gUE4ar2iR1zvIXiyq_bvfLVVjDrHwMJpef8CWipk,26235
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=cHEKA4whB4tVzZrsRoGkbdSDRgdfZA77xifYvA2XZMU,22091
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
-sws_spark_dissemination_helper/constants.py,sha256=XWbXdsgar4ZCsuhmyFB2VDLL3QMTEkXFyfYa9cYK3I8,13173
+sws_spark_dissemination_helper/constants.py,sha256=zviO6huxWTWonHv4v2M8zKr7HXCDMBGqjHx-eTfGT2A,13487
 sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
-sws_spark_dissemination_helper-0.0.160.dist-info/METADATA,sha256=5_Ju2WgV0yAErjDi0spS6aF5-hYDHIxfXsR_lrJRl7k,2824
-sws_spark_dissemination_helper-0.0.160.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.160.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.160.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.162.dist-info/METADATA,sha256=WRolvgcPvN8XgSPJ9fo0DirLLBSdEQ8b0fUhoayKSmY,2824
+sws_spark_dissemination_helper-0.0.162.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.162.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.162.dist-info/RECORD,,