sws-spark-dissemination-helper 0.0.158__tar.gz → 0.0.160__tar.gz

{sws_spark_dissemination_helper-0.0.158 → sws_spark_dissemination_helper-0.0.160}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.158
+Version: 0.0.160
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.158 → sws_spark_dissemination_helper-0.0.160}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.158"
+version = "0.0.160"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.158 → sws_spark_dissemination_helper-0.0.160}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -687,7 +687,7 @@ class SWSBronzeIcebergSparkHelper:
         else:
             self.disseminated_tag_df = self.df_bronze
 
-        if not from_tag and len(dimensions) != 0:
+        if not from_tag and dimensions is not None and len(dimensions) != 0:
             for dimension_name, codes in dimensions.items():
                 logging.info(f"dimension_name: {dimension_name}")
                 logging.info(f"codes: {codes}")

{sws_spark_dissemination_helper-0.0.158 → sws_spark_dissemination_helper-0.0.160}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -496,8 +496,8 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
         new_diss_table = BaseDisseminatedTagTable(
-            id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold pre-SDMX csv",
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
             description="Gold table containing the tag data without any processing cached in csv",
             layer=TableLayer.GOLD,
             private=True,
@@ -515,3 +515,52 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faosta_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df

{sws_spark_dissemination_helper-0.0.158 → sws_spark_dissemination_helper-0.0.160}/src/sws_spark_dissemination_helper/constants.py

@@ -274,6 +274,12 @@ class IcebergTables:
         self.GOLD_PRE_SDMX = self._create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
+        self.GOLD_FAOSTAT = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self._create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
     def _create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""