sws-spark-dissemination-helper 0.0.93__tar.gz → 0.0.95__tar.gz

{sws_spark_dissemination_helper-0.0.93 → sws_spark_dissemination_helper-0.0.95}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.93
+Version: 0.0.95
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.93 → sws_spark_dissemination_helper-0.0.95}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.93"
+version = "0.0.95"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.93 → sws_spark_dissemination_helper-0.0.95}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -447,6 +447,9 @@ class SWSBronzeIcebergSparkHelper:
     def write_bronze_disseminated_tag_data_to_iceberg_and_csv(
         self, dimensions: Dict[str, List[str]]
     ) -> DataFrame:
+        
+        refs = self.spark.sql(f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs").collect()
+        logging.info(f"bronze refs: {refs}")
 
         create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
         logging.info(f"create_branch_query: {create_branch_query}")
@@ -454,25 +457,36 @@ class SWSBronzeIcebergSparkHelper:
 
         logging.info(f"result of create_branch_query: {create_branch_query_result}")
 
+        self.disseminated_tag_df = self.spark.read.option(
+            "branch", f"`diss_tag_{self.tag_name}`"
+        ).table(self.iceberg_tables.BRONZE.iceberg_id)
+
         logging.info(f"dimensions: {dimensions}")
         for dimension_name, codes in dimensions.items():
             logging.info(f"dimension_name: {dimension_name}")
             logging.info(f"codes: {codes}")
             if len(codes) != 0:
-                not_in_codes = ",".join([f"'{code}'" for code in codes])
-                delete_from_branch_query = f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
-                logging.info(f"delete_from_branch_query: {delete_from_branch_query}")
-                delete_from_branch_query_result = self.spark.sql(
-                    delete_from_branch_query
-                ).collect()
-
-                logging.info(
-                    f"result of delete_from_branch_query: {delete_from_branch_query_result}"
+                # not_in_codes = ",".join([f"'{code}'" for code in codes])
+                # delete_from_branch_query = f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
+                # logging.info(f"delete_from_branch_query: {delete_from_branch_query}")
+                # delete_from_branch_query_result = self.spark.sql(
+                #     delete_from_branch_query
+                # ).collect()
+
+                # logging.info(
+                #     f"result of delete_from_branch_query: {delete_from_branch_query_result}"
+                # )
+                self.disseminated_tag_df = self.disseminated_tag_df.filter(
+                    col(dimension_name).isin(codes)
                 )
 
-        self.disseminated_tag_df = self.spark.read.option(
-            "branch", f"`diss_tag_{self.tag_name}`"
-        ).table(self.iceberg_tables.BRONZE.iceberg_id)
+        # self.disseminated_tag_df = self.spark.read.option(
+        #     "branch", f"`diss_tag_{self.tag_name}`"
+        # ).table(self.iceberg_tables.BRONZE.iceberg_id)
+
+        self.disseminated_tag_df.write.format("iceberg").mode("overwrite").save(
+            f"{self.iceberg_tables.BRONZE.iceberg_id}.`diss_tag_{self.tag_name}`"
+        )
 
         disseminated_tag_df = self.disseminated_tag_df.withColumn(
             "metadata", F.to_json(col("metadata"))