sws-spark-dissemination-helper 0.0.98__py3-none-any.whl → 0.0.100__py3-none-any.whl

sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -464,6 +464,7 @@ class SWSBronzeIcebergSparkHelper:
         logging.info(f"bronze refs: {refs}")
 
         create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
+        create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"
         logging.info(f"create_branch_query: {create_branch_query}")
         create_branch_query_result = self.spark.sql(create_branch_query).collect()
 
@@ -493,27 +494,27 @@ class SWSBronzeIcebergSparkHelper:
             logging.info(f"dimension_name: {dimension_name}")
             logging.info(f"codes: {codes}")
             if len(codes) != 0:
-                # not_in_codes = ",".join([f"'{code}'" for code in codes])
-                # delete_from_branch_query = f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
-                # logging.info(f"delete_from_branch_query: {delete_from_branch_query}")
-                # delete_from_branch_query_result = self.spark.sql(
-                #     delete_from_branch_query
-                # ).collect()
-
-                # logging.info(
-                #     f"result of delete_from_branch_query: {delete_from_branch_query_result}"
-                # )
-                self.disseminated_tag_df = self.disseminated_tag_df.filter(
-                    col(dimension_name).isin(codes)
+                not_in_codes = ",".join([f"'{code}'" for code in codes])
+                delete_from_branch_query = f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
+                logging.info(f"delete_from_branch_query: {delete_from_branch_query}")
+                delete_from_branch_query_result = self.spark.sql(
+                    delete_from_branch_query
+                ).collect()
+
+                logging.info(
+                    f"result of delete_from_branch_query: {delete_from_branch_query_result}"
                 )
+                # self.disseminated_tag_df = self.disseminated_tag_df.filter(
+                #     col(dimension_name).isin(codes)
+                # )
 
-        # self.disseminated_tag_df = self.spark.read.option(
-        #     "branch", f"`diss_tag_{self.tag_name}`"
-        # ).table(self.iceberg_tables.BRONZE.iceberg_id)
+        self.disseminated_tag_df = self.spark.read.option(
+            "branch", f"`diss_tag_{self.tag_name}`"
+        ).table(self.iceberg_tables.BRONZE.iceberg_id)
 
-        self.disseminated_tag_df.writeTo(
-            f"{self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}`"
-        ).overwritePartitions()
+        # self.disseminated_tag_df.writeTo(
+        #     f"{self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}`"
+        # ).overwritePartitions()
 
         disseminated_tag_df = self.disseminated_tag_df.withColumn(
             "metadata", F.to_json(col("metadata"))

{sws_spark_dissemination_helper-0.0.98.dist-info → sws_spark_dissemination_helper-0.0.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.98
+Version: 0.0.100
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.98.dist-info → sws_spark_dissemination_helper-0.0.100.dist-info}/RECORD

@@ -1,11 +1,11 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=VzJ9_DfMAxKAS_IeJYgA5L7ByNKJfL-RxXkC34tb_oQ,21954
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=vCnw4PVJ41Fiqwl0nuC-bXbJ4nqelx_EsFuraP2OAuk,22079
 sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=ZC7hxkppo6qmfCc2z5vm2Y2iH1901F-rx9Er9cxuzP4,16037
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=zEppNq5shiHZH2yt5faWGsb5QEmpAQS0ToIrG6fmv6o,22231
 sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
 sws_spark_dissemination_helper/constants.py,sha256=hpHHlbojShMWRfyIelXz6c5BqFzO48Oap1zmztlMMrs,11349
 sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.98.dist-info/METADATA,sha256=jQUUSqW2i9euAPfR77giFLWTdQ7vw04n2E1vAYKBMGk,2823
-sws_spark_dissemination_helper-0.0.98.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.98.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.98.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.100.dist-info/METADATA,sha256=8z7Q9un-K_28KTvKWQC9jhK42XaHzepACSOFycGxFRU,2824
+sws_spark_dissemination_helper-0.0.100.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.100.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.100.dist-info/RECORD,,