sws-spark-dissemination-helper 0.0.145__tar.gz → 0.0.147__tar.gz

{sws_spark_dissemination_helper-0.0.145 → sws_spark_dissemination_helper-0.0.147}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.145
+Version: 0.0.147
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.145 → sws_spark_dissemination_helper-0.0.147}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.145"
+version = "0.0.147"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3>=1.36.18",

{sws_spark_dissemination_helper-0.0.145 → sws_spark_dissemination_helper-0.0.147}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -1,6 +1,6 @@
 import logging
 from copy import copy
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import pyspark.sql.functions as F
 from pyspark.sql import DataFrame, SparkSession
@@ -26,6 +26,7 @@ class SWSEasyIcebergSparkHelper:
         dataset_tables: DatasetTables = None,
         keep_history: bool = False,
         write_csv: bool = True,
+        source_tag: Union[str, None] = None,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -37,6 +38,7 @@ class SWSEasyIcebergSparkHelper:
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.keep_history: bool = keep_history
         self.write_csv: bool = write_csv
+        self.source_tag: Union[str, None] = source_tag
 
         if dataset_details is not None:
             (
@@ -496,9 +498,23 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self, sql=False) -> DataFrame:
+    def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql_from_tag()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
+    def write_data_to_iceberg_and_csv(self, sql=False, from_tag=False) -> DataFrame:
         if sql:
             self.df_denorm = self._gen_denormalied_data_sql()
+        elif from_tag:
+            self.df_denorm = self._gen_denormalied_data_sql_from_tag()
         else:
             self.df_denorm = self._gen_denormalied_data()