sws-spark-dissemination-helper 0.0.151__tar.gz → 0.0.152__tar.gz

{sws_spark_dissemination_helper-0.0.151 → sws_spark_dissemination_helper-0.0.152}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.151
+Version: 0.0.152
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.151 → sws_spark_dissemination_helper-0.0.152}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.151"
+version = "0.0.152"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.151 → sws_spark_dissemination_helper-0.0.152}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -511,11 +511,9 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self, sql=False, from_tag=False) -> DataFrame:
+    def write_data_to_iceberg_and_csv(self, sql=True) -> DataFrame:
         if sql:
             self.df_denorm = self._gen_denormalied_data_sql()
-        elif from_tag:
-            self.df_denorm = self._gen_denormalied_data_sql_from_tag()
         else:
             self.df_denorm = self._gen_denormalied_data()
 
@@ -586,18 +584,21 @@ class SWSEasyIcebergSparkHelper:
         logging.info("Unfiltered data tags successfully written")
 
     def write_filtered_data_to_iceberg_and_csv(
-        self, dimensions: Dict[str, List[str]]
+        self, dimensions: Dict[str, List[str]] = None, from_tag=False
     ) -> DataFrame:
 
-        self.filtered_df = self.df_denorm
-
-        for dimension_name, codes in dimensions.items():
-            logging.info(f"dimension_name: {dimension_name}")
-            logging.info(f"codes: {codes}")
-            if len(codes) != 0:
-                self.filtered_df = self.filtered_df.filter(
-                    col(dimension_name).isin(codes)
-                )
+        if from_tag:
+            self.filtered_df = self._gen_denormalied_data_sql_from_tag()
+        else:
+            self.filtered_df = self.df_denorm
+
+            for dimension_name, codes in dimensions.items():
+                logging.info(f"dimension_name: {dimension_name}")
+                logging.info(f"codes: {codes}")
+                if len(codes) != 0:
+                    self.filtered_df = self.filtered_df.filter(
+                        col(dimension_name).isin(codes)
+                    )
 
         self.filtered_df.writeTo(
             self.iceberg_tables.TABLE_FILTERED.iceberg_id