sws-spark-dissemination-helper 0.0.126__tar.gz → 0.0.128__tar.gz

{sws_spark_dissemination_helper-0.0.126 → sws_spark_dissemination_helper-0.0.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.126
+Version: 0.0.128
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.126 → sws_spark_dissemination_helper-0.0.128}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.126"
+version = "0.0.128"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.126 → sws_spark_dissemination_helper-0.0.128}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -25,6 +25,7 @@ class SWSEasyIcebergSparkHelper:
         dataset_details: dict = None,
         dataset_tables: DatasetTables = None,
         keep_history: bool = False,
+        write_csv: bool = True,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -35,6 +36,7 @@ class SWSEasyIcebergSparkHelper:
         self.dataset_tables: DatasetTables = dataset_tables
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.keep_history: bool = keep_history
+        self.write_csv: bool = write_csv
 
         if dataset_details is not None:
             (
@@ -335,16 +337,16 @@ class SWSEasyIcebergSparkHelper:
 
         logging.info(f"Iceberg tag '{self.tag_name}' created")
 
-        df_denorm = self.df_denorm.withColumn(
-            "metadata", F.to_json(col("metadata"))
-        ).coalesce(1)
+        df_denorm = self.df_denorm.withColumn("metadata", F.to_json(col("metadata")))
+        if self.write_csv:
+            df_denorm = df_denorm.coalesce(1)
 
-        save_cache_csv(
-            df=df_denorm,
-            bucket=self.bucket,
-            prefix=self.iceberg_tables.TABLE.csv_prefix,
-            tag_name=self.tag_name,
-        )
+            save_cache_csv(
+                df=df_denorm,
+                bucket=self.bucket,
+                prefix=self.iceberg_tables.TABLE.csv_prefix,
+                tag_name=self.tag_name,
+            )
 
         return df_denorm
 
@@ -372,21 +374,22 @@ class SWSEasyIcebergSparkHelper:
         )
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
-        new_csv_table = BaseDisseminatedTagTable(
-            id="unfiltered_csv",
-            name="Unfiltered csv",
-            description="Csv table containing all the raw data imported from the SWS and denormalized",
-            layer=TableLayer.CACHE,
-            private=True,
-            debug=True,
-            type=TableType.CSV,
-            path=self.iceberg_tables.TABLE.csv_path,
-            structure={"columns": self.df_denorm.schema.jsonValue()["fields"]},
-        )
-        tag = tags.add_dissemination_table(
-            self.dataset_id, self.tag_name, new_csv_table
-        )
-        logging.debug(f"Tag with Added csv Table: {tag}")
+        if self.write_csv:
+            new_csv_table = BaseDisseminatedTagTable(
+                id="unfiltered_csv",
+                name="Unfiltered csv",
+                description="Csv table containing all the raw data imported from the SWS and denormalized",
+                layer=TableLayer.CACHE,
+                private=True,
+                debug=True,
+                type=TableType.CSV,
+                path=self.iceberg_tables.TABLE.csv_path,
+                structure={"columns": self.df_denorm.schema.jsonValue()["fields"]},
+            )
+            tag = tags.add_dissemination_table(
+                self.dataset_id, self.tag_name, new_csv_table
+            )
+            logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Unfiltered data tags successfully written")
 
@@ -418,18 +421,21 @@ class SWSEasyIcebergSparkHelper:
 
         disseminated_tag_df = self.filtered_df.withColumn(
             "metadata", F.to_json(col("metadata"))
-        ).coalesce(1)
-
-        save_cache_csv(
-            df=disseminated_tag_df,
-            bucket=self.bucket,
-            prefix=f"{self.iceberg_tables.TABLE_FILTERED.csv_prefix}",
-            tag_name=self.tag_name,
         )
 
+        if self.write_csv:
+            disseminated_tag_df = disseminated_tag_df.coalesce(1)
+
+            save_cache_csv(
+                df=disseminated_tag_df,
+                bucket=self.bucket,
+                prefix=f"{self.iceberg_tables.TABLE_FILTERED.csv_prefix}",
+                tag_name=self.tag_name,
+            )
+
         return disseminated_tag_df
 
-    def write_bronze_sws_filtered_disseminated_tag(self, tags: Tags):
+    def write_sws_filtered_dissemination_tag(self, tags: Tags):
         # Get or create a new tag
         tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
         logging.debug(f"Tag: {tag}")
@@ -452,20 +458,21 @@ class SWSEasyIcebergSparkHelper:
         )
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
-        new_csv_table = BaseDisseminatedTagTable(
-            id="filtered_csv",
-            name="Filtered csv",
-            description="Csv table containing the raw data imported from the SWS, denormalized and filtered per dimension cached",
-            layer=TableLayer.CACHE,
-            private=True,
-            type=TableType.CSV,
-            path=self.iceberg_tables.TABLE_FILTERED.csv_path,
-            structure={"columns": self.filtered_df.schema.jsonValue()["fields"]},
-        )
-        tag = tags.add_dissemination_table(
-            self.dataset_id, self.tag_name, new_csv_table
-        )
+        if self.write_csv:
+            new_csv_table = BaseDisseminatedTagTable(
+                id="filtered_csv",
+                name="Filtered csv",
+                description="Csv table containing the raw data imported from the SWS, denormalized and filtered per dimension cached",
+                layer=TableLayer.CACHE,
+                private=True,
+                type=TableType.CSV,
+                path=self.iceberg_tables.TABLE_FILTERED.csv_path,
+                structure={"columns": self.filtered_df.schema.jsonValue()["fields"]},
+            )
+            tag = tags.add_dissemination_table(
+                self.dataset_id, self.tag_name, new_csv_table
+            )
 
-        logging.debug(f"Tag with Added csv Table: {tag}")
+            logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")