sws-spark-dissemination-helper 0.0.101__tar.gz → 0.0.103__tar.gz

{sws_spark_dissemination_helper-0.0.101 → sws_spark_dissemination_helper-0.0.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.101
+Version: 0.0.103
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.101 → sws_spark_dissemination_helper-0.0.103}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.101"
+version = "0.0.103"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.101 → sws_spark_dissemination_helper-0.0.103}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -387,18 +387,12 @@ class SWSBronzeIcebergSparkHelper:
 
         logging.info(f"Bronze table written to {self.iceberg_tables.BRONZE.iceberg_id}")
 
+        self.spark.sql(
+            f"ALTER ICEBERG TABLE {self.iceberg_tables.BRONZE.iceberg_id} REFRESH;"
+        )
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE TAG `{self.tag_name}`"
         )
-        while (
-            self.spark.sql(
-                f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
-            )
-            .filter((col("type") == lit("TAG")) & (col("name") == lit(self.tag_name)))
-            .count()
-        ) == 0:
-            logging.info(f"Waiting for the tag {self.tag_name} to be created")
-            time.sleep(2)
 
         logging.info(f"bronze tag '{self.tag_name}' created")
 
@@ -476,6 +470,9 @@ class SWSBronzeIcebergSparkHelper:
             f"Bronze disseminated tag table written to {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id}"
         )
 
+        self.spark.sql(
+            f"ALTER ICEBERG TABLE {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id} REFRESH;"
+        )
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id} CREATE TAG `{self.tag_name}`"
         )

{sws_spark_dissemination_helper-0.0.101 → sws_spark_dissemination_helper-0.0.103}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -110,9 +110,10 @@ class SWSSilverIcebergSparkHelper:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.BRONZE.iceberg_id
         )
+
     def read_bronze_diss_tag_data(self) -> DataFrame:
-        return self.spark.read.option("branch", f"diss_tag_{self.tag_name}").table(
-            self.iceberg_tables.BRONZE.iceberg_id
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
         )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]: