sws-spark-dissemination-helper 0.0.97__tar.gz → 0.0.99__tar.gz

{sws_spark_dissemination_helper-0.0.97 → sws_spark_dissemination_helper-0.0.99}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.97
+Version: 0.0.99
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.97 → sws_spark_dissemination_helper-0.0.99}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.97"
+version = "0.0.99"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.97 → sws_spark_dissemination_helper-0.0.99}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -464,6 +464,7 @@ class SWSBronzeIcebergSparkHelper:
         logging.info(f"bronze refs: {refs}")
 
         create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
+        create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"
         logging.info(f"create_branch_query: {create_branch_query}")
         create_branch_query_result = self.spark.sql(create_branch_query).collect()
 
@@ -511,9 +512,9 @@ class SWSBronzeIcebergSparkHelper:
         #     "branch", f"`diss_tag_{self.tag_name}`"
         # ).table(self.iceberg_tables.BRONZE.iceberg_id)
 
-        self.disseminated_tag_df.write.format("iceberg").mode("overwrite").save(
+        self.disseminated_tag_df.writeTo(
             f"{self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}`"
-        )
+        ).overwritePartitions()
 
         disseminated_tag_df = self.disseminated_tag_df.withColumn(
             "metadata", F.to_json(col("metadata"))