sws-spark-dissemination-helper 0.0.99__tar.gz → 0.0.101__tar.gz

{sws_spark_dissemination_helper-0.0.99 → sws_spark_dissemination_helper-0.0.101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.99
+Version: 0.0.101
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.99 → sws_spark_dissemination_helper-0.0.101}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.99"
+version = "0.0.101"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.99 → sws_spark_dissemination_helper-0.0.101}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -458,63 +458,27 @@ class SWSBronzeIcebergSparkHelper:
         self, dimensions: Dict[str, List[str]]
     ) -> DataFrame:
 
-        refs = self.spark.sql(
-            f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
-        ).collect()
-        logging.info(f"bronze refs: {refs}")
+        self.disseminated_tag_df = self.df_bronze
 
-        create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
-        create_branch_query = f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id} CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"
-        logging.info(f"create_branch_query: {create_branch_query}")
-        create_branch_query_result = self.spark.sql(create_branch_query).collect()
-
-        while (
-            self.spark.sql(
-                f"SELECT * FROM {self.iceberg_tables.BRONZE.iceberg_id}.refs"
-            )
-            .filter(
-                (col("type") == lit("BRANCH"))
-                & (col("name") == lit(f"diss_tag_{self.tag_name}"))
-            )
-            .count()
-        ) == 0:
-            logging.info(
-                f"Waiting for the branch {self.tag_name} diss_tag_{self.tag_name} to be created"
-            )
-            time.sleep(2)
-
-        logging.info(f"result of create_branch_query: {create_branch_query_result}")
-
-        self.disseminated_tag_df = self.spark.read.option(
-            "branch", f"`diss_tag_{self.tag_name}`"
-        ).table(self.iceberg_tables.BRONZE.iceberg_id)
-
-        logging.info(f"dimensions: {dimensions}")
         for dimension_name, codes in dimensions.items():
             logging.info(f"dimension_name: {dimension_name}")
             logging.info(f"codes: {codes}")
             if len(codes) != 0:
-                # not_in_codes = ",".join([f"'{code}'" for code in codes])
-                # delete_from_branch_query = f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
-                # logging.info(f"delete_from_branch_query: {delete_from_branch_query}")
-                # delete_from_branch_query_result = self.spark.sql(
-                #     delete_from_branch_query
-                # ).collect()
-
-                # logging.info(
-                #     f"result of delete_from_branch_query: {delete_from_branch_query_result}"
-                # )
                 self.disseminated_tag_df = self.disseminated_tag_df.filter(
                     col(dimension_name).isin(codes)
                 )
 
-        # self.disseminated_tag_df = self.spark.read.option(
-        #     "branch", f"`diss_tag_{self.tag_name}`"
-        # ).table(self.iceberg_tables.BRONZE.iceberg_id)
+        self.df_bronze.writeTo(
+            self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Bronze disseminated tag table written to {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id}"
+        )
 
-        self.disseminated_tag_df.writeTo(
-            f"{self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}`"
-        ).overwritePartitions()
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.BRONZE_DISS_TAG.iceberg_id} CREATE TAG `{self.tag_name}`"
+        )
 
         disseminated_tag_df = self.disseminated_tag_df.withColumn(
             "metadata", F.to_json(col("metadata"))
@@ -523,7 +487,7 @@ class SWSBronzeIcebergSparkHelper:
         save_cache_csv(
             df=disseminated_tag_df,
             bucket=self.bucket,
-            prefix=f"{self.iceberg_tables.BRONZE.csv_prefix}_disseminated_tag",
+            prefix=f"{self.iceberg_tables.BRONZE_DISS_TAG.csv_prefix}",
             tag_name=self.tag_name,
         )
 
@@ -542,8 +506,8 @@ class SWSBronzeIcebergSparkHelper:
             private=True,
             type=TableType.ICEBERG,
             database=IcebergDatabases.BRONZE_DATABASE,
-            table=self.iceberg_tables.BRONZE.table,
-            path=self.iceberg_tables.BRONZE.path,
+            table=self.iceberg_tables.BRONZE_DISS_TAG.table,
+            path=self.iceberg_tables.BRONZE_DISS_TAG.path,
             structure={
                 "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
             },
@@ -561,7 +525,7 @@ class SWSBronzeIcebergSparkHelper:
             private=True,
             type=TableType.CSV,
             # TODO Correct the path in the origin library
-            path=self.iceberg_tables.BRONZE.csv_path,
+            path=self.iceberg_tables.BRONZE_DISS_TAG.csv_path,
             structure={
                 "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
             },

{sws_spark_dissemination_helper-0.0.99 → sws_spark_dissemination_helper-0.0.101}/src/sws_spark_dissemination_helper/constants.py

@@ -219,6 +219,7 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         self.BRONZE = self._create_iceberg_table("BRONZE")
+        self.BRONZE_DISS_TAG = self._create_iceberg_table("BRONZE", suffix="diss_tag")
         self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes