sws-spark-dissemination-helper 0.0.101__py3-none-any.whl → 0.0.103__py3-none-any.whl

sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -387,18 +387,12 @@ class SWSBronzeIcebergSparkHelper:
 
         logging.info(f"Bronze table written to {self.iceberg_tables.BRONZE.iceberg_id}")
 
+        self.spark.sql(
+            f"ALTER ICEBERG TABLE {self.iceberg_tables.BRONZE.iceberg_id} REFRESH;"
+        )
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE TAG `{self.tag_name}`"
         )
-        while (
-            self.spark.sql(
-                f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
-            )
-            .filter((col("type") == lit("TAG")) & (col("name") == lit(self.tag_name)))
-            .count()
-        ) == 0:
-            logging.info(f"Waiting for the tag {self.tag_name} to be created")
-            time.sleep(2)
 
         logging.info(f"bronze tag '{self.tag_name}' created")
 
@@ -476,6 +470,9 @@ class SWSBronzeIcebergSparkHelper:
             f"Bronze disseminated tag table written to {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id}"
         )
 
+        self.spark.sql(
+            f"ALTER ICEBERG TABLE {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id} REFRESH;"
+        )
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id} CREATE TAG `{self.tag_name}`"
         )

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -110,9 +110,10 @@ class SWSSilverIcebergSparkHelper:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.BRONZE.iceberg_id
         )
+
     def read_bronze_diss_tag_data(self) -> DataFrame:
-        return self.spark.read.option("branch", f"diss_tag_{self.tag_name}").table(
-            self.iceberg_tables.BRONZE.iceberg_id
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
         )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:

{sws_spark_dissemination_helper-0.0.101.dist-info → sws_spark_dissemination_helper-0.0.103.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.101
+Version: 0.0.103
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.101.dist-info → sws_spark_dissemination_helper-0.0.103.dist-info}/RECORD

@@ -1,11 +1,11 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=k79yVW14wnNTX1nLEZj4bqsmv9L43Hd5ILuAxFWTX2s,20146
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=KqRX5yl3rk7qntuxtPSdKoPFwSbzeyp3HjHRB3JUaaE,20026
 sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=ZC7hxkppo6qmfCc2z5vm2Y2iH1901F-rx9Er9cxuzP4,16037
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=zEppNq5shiHZH2yt5faWGsb5QEmpAQS0ToIrG6fmv6o,22231
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=08xA7h4oVicKOnjEoMrLXpuk9mZ2GTiTPx3C7rCI7pg,22224
 sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
 sws_spark_dissemination_helper/constants.py,sha256=yG_pUi2PVeFKz6pogAU2AGQ-wV4FHWRHOW77wE_XGL0,11436
 sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.101.dist-info/METADATA,sha256=wDEhvo5aNx0XYvV1kqUsG5UdPJ6rIDp8Hbk1zXvzCk8,2824
-sws_spark_dissemination_helper-0.0.101.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.101.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.101.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.103.dist-info/METADATA,sha256=Bfx-ejCQkuh116Ip3YMIuBnhG3B1bJp7JWyOhIi2mLA,2824
+sws_spark_dissemination_helper-0.0.103.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.103.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.103.dist-info/RECORD,,