sws-spark-dissemination-helper 0.0.161__py3-none-any.whl → 0.0.163__py3-none-any.whl

sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -510,6 +510,10 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_denormalied_data_sql(self) -> DataFrame:
+        self._gen_denormalized_observation_sql()
+        self._gen_denormalized_metadata_sql()
+        self._gen_grouped_metadata_sql()
+
         return self.spark.sql(
             f"""
             SELECT

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -271,6 +271,31 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:

{sws_spark_dissemination_helper-0.0.161.dist-info → sws_spark_dissemination_helper-0.0.163.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.161
+Version: 0.0.163
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.161.dist-info → sws_spark_dissemination_helper-0.0.163.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=ocuau0WtpyRwui0qwdQ_Rxh4nYPOyZoHpGKaWRa6B3Q,28868
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=Ko_gUE4ar2iR1zvIXiyq_bvfLVVjDrHwMJpef8CWipk,26235
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=RcpU8znoyNOfJMLFwrgi7S-BKkzgqh25o2wPLNIxLYc,21246
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=abNk2rsBvDcHESczU07Ouc731HH69ypFEXA0PuvxOYM,26372
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=cHEKA4whB4tVzZrsRoGkbdSDRgdfZA77xifYvA2XZMU,22091
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
 sws_spark_dissemination_helper/constants.py,sha256=zviO6huxWTWonHv4v2M8zKr7HXCDMBGqjHx-eTfGT2A,13487
 sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
-sws_spark_dissemination_helper-0.0.161.dist-info/METADATA,sha256=62_Bd5pfNPWI4hWnflkj1HbyL33bdG_c8GbGp7SLY3U,2824
-sws_spark_dissemination_helper-0.0.161.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.161.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.161.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.163.dist-info/METADATA,sha256=j3TwgkmurtYUIOg6LQ83EelFI88FyUTK1g4hOTsareU,2824
+sws_spark_dissemination_helper-0.0.163.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.163.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.163.dist-info/RECORD,,