sws-spark-dissemination-helper 0.0.95__tar.gz → 0.0.96__tar.gz

{sws_spark_dissemination_helper-0.0.95 → sws_spark_dissemination_helper-0.0.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.95
+Version: 0.0.96
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.95 → sws_spark_dissemination_helper-0.0.96}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.95"
+version = "0.0.96"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.95 → sws_spark_dissemination_helper-0.0.96}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -1,4 +1,5 @@
 import logging
+import time
 from copy import copy
 from typing import Dict, List, Tuple
 
@@ -389,6 +390,15 @@ class SWSBronzeIcebergSparkHelper:
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE TAG `{self.tag_name}`"
         )
+        while (
+            self.spark.sql(
+                f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
+            )
+            .filter((col("type") == lit("TAG")) & (col("name") == lit(self.tag_name)))
+            .count()
+        ) == 0:
+            logging.info(f"Waiting for the tag {self.tag_name} to be created")
+            time.sleep(2)
 
         logging.info(f"bronze tag '{self.tag_name}' created")
 
@@ -447,14 +457,31 @@ class SWSBronzeIcebergSparkHelper:
     def write_bronze_disseminated_tag_data_to_iceberg_and_csv(
         self, dimensions: Dict[str, List[str]]
     ) -> DataFrame:
-        
-        refs = self.spark.sql(f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs").collect()
+
+        refs = self.spark.sql(
+            f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
+        ).collect()
         logging.info(f"bronze refs: {refs}")
 
         create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
         logging.info(f"create_branch_query: {create_branch_query}")
         create_branch_query_result = self.spark.sql(create_branch_query).collect()
 
+        while (
+            self.spark.sql(
+                f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
+            )
+            .filter(
+                (col("type") == lit("BRANCH"))
+                & (col("name") == lit(f"diss_tag_{self.tag_name}"))
+            )
+            .count()
+        ) == 0:
+            logging.info(
+                f"Waiting for the branch {self.tag_name} diss_tag_{self.tag_name} to be created"
+            )
+            time.sleep(2)
+
         logging.info(f"result of create_branch_query: {create_branch_query_result}")
 
         self.disseminated_tag_df = self.spark.read.option(